diff --git a/.dockerignore b/.dockerignore index a4bf91f5486fa1119db756a422891fbb1ac5526c..669b1c7d279b1d2bfd6bd6b8da5102f18cbf1583 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,3 +5,4 @@ !/go.sum !/resource-server-nginx.conf !/version +!/docker diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 7df2129a40ffe98d816a142205058b56487ed5cd..353051391eb047c8934a3c77ed23cf3bc6116ff5 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -7,12 +7,66 @@ assignees: jamsman94 --- -**What steps did you take and what happened:** -[A clear and concise description of what the bug is.] + -**What did you expect to happen:** +**What happened?** + + +**What did you expect to happen?** + + +**How To Reproduce it(as minimally and precisely as possible)** + + +**Install Methods** + +- [ ] Helm +- [ ] Script base on K8s +- [ ] All in One +- [ ] Offline + +**Versions Used** +zadig: + +kubernetes: + + +**Environment** + +Cloud Provider: + + +Resources: + + +OS: + + +**Services Status** +```console +kubectl version +kubectl get po -n `zadig-installed-namespace` +# paste output here +``` + +If there is abnormal service, please provide service log + +```console +kubectl describe pods `abnormal-pod` +kubectl logs --tail=500 `abnormal-pod` +# paste output here +``` -**Anything else you would like to add:** -[Miscellaneous information that will assist in solving the issue.] diff --git a/.github/ISSUE_TEMPLATE/enhancement.md b/.github/ISSUE_TEMPLATE/enhancement.md index 52c0c74d2661c0caee666038a037798eb5a0e5d6..3e19b04166fdd1073d1e5419d866f77eee0f861d 100644 --- a/.github/ISSUE_TEMPLATE/enhancement.md +++ b/.github/ISSUE_TEMPLATE/enhancement.md @@ -1,7 +1,7 @@ --- name: Enhancement about: Suggest enhancements to existing features -title: "[enhencement]" +title: "[enhancement]" labels: enhancement assignees: lilianzhu diff --git a/.github/labeler.yml b/.github/labeler.yml index fa949ecb21df89c3073573b97bb04f7abd9724cb..e1d4eb4e7f073d5b3bf991d6470710297d2c4b3e 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -2,6 +2,18 @@ service/aslan: - pkg/microservice/aslan/**/* +service/picket: +- pkg/microservice/picket/**/* + +service/user: +- pkg/microservice/user/**/* + +service/policy: +- pkg/microservice/policy/**/* + +service/config: +- pkg/microservice/systemconfig/**/* + service/cron: - pkg/microservice/cron/**/* @@ -21,7 +33,10 @@ service/hubserver: - pkg/microservice/hubserver/**/* service/hubagent: - - pkg/microservice/hubagent/**/* +- pkg/microservice/hubagent/**/* + +service/upgradeassistant: +- pkg/microservice/upgradeassistant/**/* common: - pkg/config/**/* diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 99528951a8c863787fe84435522fb6c1e3674432..00ff9d193bfa56179e498c2a83678da46159ae19 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,22 +1,13 @@ ### What this PR does / Why we need it: -Issue Number: close #xxx - -Problem Summary: ### What is changed and how it works? -What's Changed: - -How it Works: - -### Check List - - -- [ ] Docs have been added / updated -- [ ] Unit test / Integration test for the changes have been added -- [ ] Manual test (add detailed scripts or steps below) -- [ ] No code +### Does this PR introduce a user-facing change? -## More information \ No newline at end of file +- [ ] API change +- [ ] database schema change +- [ ] behavioral change +- [ ] change in non-functional attributes such as efficiency or availability +- [ ] fix of a previous issue diff --git a/CONTRIBUTING-zh-CN.md b/CONTRIBUTING-zh-CN.md index fb156f68377dd5d2295925b53bb560c12cfb3f9f..0745307c1541302f9996c625f11147d196c35375 100644 --- a/CONTRIBUTING-zh-CN.md +++ b/CONTRIBUTING-zh-CN.md @@ -167,4 +167,3 @@ swag init -d ./lib/microservice/aslan/ -g server/rest/router.go -o ./lib/microse ### 其它资源 - [项目的 maintainers](GOVERNANCE.md#maintainers) -- [API 文档](https://os.koderover.com/api/spock/apidocs/index.html) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 24744da4ef0638284284404ed4ed2a584490054d..0631c88234d0bce47704d85c8703a900e2a23f5b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -180,4 +180,3 @@ more information. ### More resources - [Current maintainers](GOVERNANCE.md#maintainers) -- [API docs](https://os.koderover.com/api/spock/apidocs/index.html) diff --git a/GOVERNANCE.md b/GOVERNANCE.md index 368ccfb3f0b42a641fe344a835136d2603305d1a..cac629d6ef07870a8e65fecc0b1bafc8b3d80954 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -60,4 +60,16 @@ The council will make a decision for the nomination. The candidate has to have d - [liuflylove666](https://github.com/liuflylove666) - [jiangpeipei327](https://github.com/jiangpeipei327) - [lyzhang1999](https://github.com/lyzhang1999) -- [avtion](https://github.com/avtion) \ No newline at end of file +- [avtion](https://github.com/avtion) +- [wbjxxzx](https://github.com/wbjxxzx) +- [jtr109](https://github.com/jtr109) +- [huangzexiong](https://github.com/huangzexiong) +- [ruchikabobade](https://github.com/ruchikabobade) +- [fengzixu](https://github.com/fengzixu) +- [xrfinbupt](https://github.com/xrfinbupt) +- [mhatvan](https://github.com/mhatvan) +- [tbbrave](https://github.com/tbbrave) +- [lxm](https://github.com/lxm) +- [zxdvd](https://github.com/zxdvd) +- [rovast](https://github.com/rovast) +- [pokixing](https://github.com/pokixing) \ No newline at end of file diff --git a/Makefile b/Makefile index d46ca9fbc2fdebec8c2b184020381599c9689dda..0341b1bb175b8c5b712bab8781c7c7138af70b78 100644 --- a/Makefile +++ b/Makefile @@ -1,74 +1,39 @@ # New Makefile for multi-architecture .PHONY: all -IMAGE_REPOSITORY = ccr.ccs.tencentyun.com/koderover-rc +IMAGE_REPOSITORY = koderover.tencentcloudcr.com/koderover-public VERSION ?= $(shell date +'%Y%m%d%H%M%S') - -TARGETS = aslan cron hub-agent hub-server jenkins-plugin podexec predator-plugin resource-server ua warpdrive policy picket +VERSION := $(VERSION) +TARGETS = aslan config cron hub-agent hub-server init jenkins-plugin packager-plugin picket podexec policy predator-plugin ua user warpdrive REAPER_OS= focal xenial bionic ALL_IMAGES=$(TARGETS:=.image) ALL_PUSH=$(TARGETS:=.push) -ALL_REAPER=$(REAPER_OS:=.reaper.image) -ALL_REAPER_PUSH=$(REAPER_OS:=.reaper.push) - -all: $(ALL_IMAGES:=.amd64) $(ALL_IMAGES:=.arm64) $(ALL_REAPER:=.amd64) $(ALL_REAPER:=.arm64) -all.push: $(ALL_PUSH:=.amd64) $(ALL_PUSH:=.arm64) $(ALL_REAPER_PUSH:=.amd64) $(ALL_REAPER_PUSH:=.arm64) -all.amd64: $(ALL_IMAGES:=.amd64) $(ALL_REAPER:=.amd64) -all.arm64: $(ALL_IMAGES:=.amd64) $(ALL_REAPER:=.arm64) -allpush.amd64: $(ALL_PUSH:=.amd64) $(ALL_REAPER_PUSH:=.amd64) -allpush.arm64: $(ALL_PUSH:=.arm64) $(ALL_REAPER_PUSH:=.arm64) +PLATFORMS=darwin linux windows +ARCHITECTURES=amd64 arm64 -%.reaper.push.arm64: MAKE_IMAGE = ${IMAGE_REPOSITORY}/reaper-plugin:${VERSION}-arm64-$* -%.reaper.push.arm64: %.reaper.image.arm64 - @docker push ${MAKE_IMAGE} +all: $(ALL_IMAGES:=.amd64) $(ALL_IMAGES:=.arm64) resource-server.build.amd64 resource-server.build.arm64 build-zgctl-all-platforms zgctl-sidecar.build.amd64 zadig-debug.build.amd64 +all.push: $(ALL_PUSH:=.amd64) $(ALL_PUSH:=.arm64) resource-server.upload.amd64 resource-server.upload.arm64 zgctl-sidecar.upload.amd64 zadig-debug.upload.amd64 -%.reaper.image.arm64: MAKE_IMAGE = ${IMAGE_REPOSITORY}/reaper-plugin:${VERSION}-arm64-$* -%.reaper.image.arm64: - @mkdir -p docker/dist/arm64 - @cp docker/service/reaper-plugin-$*.Dockerfile docker/dist/arm64/reaper-plugin-$*.Dockerfile - @sed -i -e '/#golang.Dockerfile/ {' -e 'r docker/base/arm64/golang.Dockerfile' -e 'd' -e '}' docker/dist/arm64/reaper-plugin-$*.Dockerfile - @sed -i -e '/#ubuntu-xenial.Dockerfile/ {' -e 'r docker/base/arm64/ubuntu-xenial.Dockerfile' -e 'd' -e '}' docker/dist/arm64/reaper-plugin-$*.Dockerfile - @sed -i -e '/#ubuntu-focal.Dockerfile/ {' -e 'r docker/base/arm64/ubuntu-focal.Dockerfile' -e 'd' -e '}' docker/dist/arm64/reaper-plugin-$*.Dockerfile - @sed -i -e '/#ubuntu-bionic.Dockerfile/ {' -e 'r docker/base/arm64/ubuntu-bionic.Dockerfile' -e 'd' -e '}' docker/dist/arm64/reaper-plugin-$*.Dockerfile - @sed -i -e '/#alpine.Dockerfile/ {' -e 'r docker/base/arm64/alpine.Dockerfile' -e 'd' -e '}' docker/dist/arm64/reaper-plugin-$*.Dockerfile - @sed -i -e '/#nginx.Dockerfile/ {' -e 'r docker/base/arm64/nginx.Dockerfile' -e 'd' -e '}' docker/dist/arm64/reaper-plugin-$*.Dockerfile - @sed -i -e '/#alpine-git.Dockerfile/ {' -e 'r docker/base/arm64/alpine-git.Dockerfile' -e 'd' -e '}' docker/dist/arm64/reaper-plugin-$*.Dockerfile - @docker build -f docker/dist/arm64/reaper-plugin-$*.Dockerfile --tag ${MAKE_IMAGE} . - -%.reaper.push.amd64: MAKE_IMAGE = ${IMAGE_REPOSITORY}/reaper-plugin:${VERSION}-amd64-$* -%.reaper.push.amd64: %.reaper.image.amd64 - @docker push ${MAKE_IMAGE} - -%.reaper.image.amd64: MAKE_IMAGE = ${IMAGE_REPOSITORY}/reaper-plugin:${VERSION}-amd64-$* -%.reaper.image.amd64: - @mkdir -p docker/dist/amd64 - @cp docker/service/reaper-plugin-$*.Dockerfile docker/dist/amd64/reaper-plugin-$*.Dockerfile - @sed -i -e '/#golang.Dockerfile/ {' -e 'r docker/base/amd64/golang.Dockerfile' -e 'd' -e '}' docker/dist/amd64/reaper-plugin-$*.Dockerfile - @sed -i -e '/#ubuntu-xenial.Dockerfile/ {' -e 'r docker/base/amd64/ubuntu-xenial.Dockerfile' -e 'd' -e '}' docker/dist/amd64/reaper-plugin-$*.Dockerfile - @sed -i -e '/#ubuntu-focal.Dockerfile/ {' -e 'r docker/base/amd64/ubuntu-focal.Dockerfile' -e 'd' -e '}' docker/dist/amd64/reaper-plugin-$*.Dockerfile - @sed -i -e '/#ubuntu-bionic.Dockerfile/ {' -e 'r docker/base/amd64/ubuntu-bionic.Dockerfile' -e 'd' -e '}' docker/dist/amd64/reaper-plugin-$*.Dockerfile - @sed -i -e '/#alpine.Dockerfile/ {' -e 'r docker/base/amd64/alpine.Dockerfile' -e 'd' -e '}' docker/dist/amd64/reaper-plugin-$*.Dockerfile - @sed -i -e '/#nginx.Dockerfile/ {' -e 'r docker/base/amd64/nginx.Dockerfile' -e 'd' -e '}' docker/dist/amd64/reaper-plugin-$*.Dockerfile - @sed -i -e '/#alpine-git.Dockerfile/ {' -e 'r docker/base/amd64/alpine-git.Dockerfile' -e 'd' -e '}' docker/dist/amd64/reaper-plugin-$*.Dockerfile - @docker build -f docker/dist/amd64/reaper-plugin-$*.Dockerfile --tag ${MAKE_IMAGE} . +all.amd64: $(ALL_IMAGES:=.amd64) resource-server.build.amd64 zgctl-sidecar.build.amd64 zadig-debug.build.amd64 +all.arm64: $(ALL_IMAGES:=.amd64) resource-server.build.arm64 +allpush.amd64: $(ALL_PUSH:=.amd64) $(ALL_REAPER_PUSH:=.amd64) resource-server.upload.amd64 zgctl-sidecar.upload.amd64 zadig-debug.upload.amd64 +allpush.arm64: $(ALL_PUSH:=.arm64) $(ALL_REAPER_PUSH:=.arm64) resource-server.upload.arm64 %.push.amd64: MAKE_IMAGE ?= ${IMAGE_REPOSITORY}/$*:${VERSION}-amd64 %.push.amd64: %.image.amd64 - docker push ${MAKE_IMAGE} + @docker push ${MAKE_IMAGE} %.image.amd64: MAKE_IMAGE ?= ${IMAGE_REPOSITORY}/$*:${VERSION}-amd64 %.image.amd64: @mkdir -p docker/dist/amd64 @cp docker/service/$*.Dockerfile docker/dist/amd64/$*.Dockerfile - @sed -i -e '/#golang.Dockerfile/ {' -e 'r docker/base/amd64/golang.Dockerfile' -e 'd' -e '}' docker/dist/amd64/$*.Dockerfile - @sed -i -e '/#ubuntu-xenial.Dockerfile/ {' -e 'r docker/base/amd64/ubuntu-xenial.Dockerfile' -e 'd' -e '}' docker/dist/amd64/$*.Dockerfile - @sed -i -e '/#ubuntu-bionic.Dockerfile/ {' -e 'r docker/base/amd64/ubuntu-bionic.Dockerfile' -e 'd' -e '}' docker/dist/amd64/$*.Dockerfile - @sed -i -e '/#ubuntu-focal.Dockerfile/ {' -e 'r docker/base/amd64/ubuntu-focal.Dockerfile' -e 'd' -e '}' docker/dist/amd64/$*.Dockerfile - @sed -i -e '/#alpine.Dockerfile/ {' -e 'r docker/base/amd64/alpine.Dockerfile' -e 'd' -e '}' docker/dist/amd64/$*.Dockerfile - @sed -i -e '/#nginx.Dockerfile/ {' -e 'r docker/base/amd64/nginx.Dockerfile' -e 'd' -e '}' docker/dist/amd64/$*.Dockerfile @sed -i -e '/#alpine-git.Dockerfile/ {' -e 'r docker/base/amd64/alpine-git.Dockerfile' -e 'd' -e '}' docker/dist/amd64/$*.Dockerfile + @sed -i -e '/#alpine-curl.Dockerfile/ {' -e 'r docker/base/amd64/alpine-curl.Dockerfile' -e 'd' -e '}' docker/dist/amd64/$*.Dockerfile + @sed -i -e '/#alpine.Dockerfile/ {' -e 'r docker/base/amd64/alpine.Dockerfile' -e 'd' -e '}' docker/dist/amd64/$*.Dockerfile + @sed -i -e '/#ubuntu-xenial.Dockerfile/ {' -e 'r docker/base/amd64/ubuntu-xenial.Dockerfile' -e 'd' -e '}' docker/dist/amd64/$*.Dockerfile + @CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o docker/dist/$* cmd/$*/main.go @docker build -f docker/dist/amd64/$*.Dockerfile --tag ${MAKE_IMAGE} . %.push.arm64: MAKE_IMAGE ?= ${IMAGE_REPOSITORY}/$*:${VERSION}-amd64 @@ -79,15 +44,63 @@ allpush.arm64: $(ALL_PUSH:=.arm64) $(ALL_REAPER_PUSH:=.arm64) %.image.arm64: @mkdir -p docker/dist/arm64 @cp docker/service/$*.Dockerfile docker/dist/arm64/$*.Dockerfile - @sed -i -e '/#golang.Dockerfile/ {' -e 'r docker/base/arm64/golang.Dockerfile' -e 'd' -e '}' docker/dist/arm64/$*.Dockerfile - @sed -i -e '/#ubuntu-xenial.Dockerfile/ {' -e 'r docker/base/arm64/ubuntu-xenial.Dockerfile' -e 'd' -e '}' docker/dist/arm64/$*.Dockerfile - @sed -i -e '/#ubuntu-focal.Dockerfile/ {' -e 'r docker/base/arm64/ubuntu-focal.Dockerfile' -e 'd' -e '}' docker/dist/arm64/$*.Dockerfile - @sed -i -e '/#ubuntu-bionic.Dockerfile/ {' -e 'r docker/base/arm64/ubuntu-bionic.Dockerfile' -e 'd' -e '}' docker/dist/arm64/$*.Dockerfile - @sed -i -e '/#alpine.Dockerfile/ {' -e 'r docker/base/arm64/alpine.Dockerfile' -e 'd' -e '}' docker/dist/arm64/$*.Dockerfile - @sed -i -e '/#nginx.Dockerfile/ {' -e 'r docker/base/arm64/nginx.Dockerfile' -e 'd' -e '}' docker/dist/arm64/$*.Dockerfile @sed -i -e '/#alpine-git.Dockerfile/ {' -e 'r docker/base/arm64/alpine-git.Dockerfile' -e 'd' -e '}' docker/dist/arm64/$*.Dockerfile + @sed -i -e '/#alpine-curl.Dockerfile/ {' -e 'r docker/base/arm64/alpine-curl.Dockerfile' -e 'd' -e '}' docker/dist/arm64/$*.Dockerfile + @sed -i -e '/#alpine.Dockerfile/ {' -e 'r docker/base/arm64/alpine.Dockerfile' -e 'd' -e '}' docker/dist/arm64/$*.Dockerfile + @sed -i -e '/#ubuntu-xenial.Dockerfile/ {' -e 'r docker/base/arm64/ubuntu-xenial.Dockerfile' -e 'd' -e '}' docker/dist/arm64/$*.Dockerfile + @CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -v -o docker/dist/$* cmd/$*/main.go @docker build -f docker/dist/arm64/$*.Dockerfile --tag ${MAKE_IMAGE} . +# USING BUILD AND UPLOAD INSTEAD OF IMAGE AND PUSH TO AVOID COLLISION +resource-server.build.amd64: MAKE_IMAGE ?= ${IMAGE_REPOSITORY}/resource-server:${VERSION}-amd64 +resource-server.build.amd64: + @mkdir -p docker/dist/amd64 + @cp docker/service/resource-server.Dockerfile docker/dist/amd64/resource-server.Dockerfile + @sed -i -e '/#nginx.Dockerfile/ {' -e 'r docker/base/amd64/nginx.Dockerfile' -e 'd' -e '}' docker/dist/amd64/resource-server.Dockerfile + @CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o docker/dist/reaper cmd/reaper/main.go + @CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o docker/dist/jobexecutor cmd/jobexecutor/main.go + @docker build -f docker/dist/amd64/resource-server.Dockerfile --tag ${MAKE_IMAGE} . + +resource-server.upload.amd64: MAKE_IMAGE ?= ${IMAGE_REPOSITORY}/resource-server:${VERSION}-amd64 +resource-server.upload.amd64: resource-server.build.amd64 + @docker push ${MAKE_IMAGE} + +resource-server.build.arm64: MAKE_IMAGE ?= ${IMAGE_REPOSITORY}/resource-server:${VERSION}-arm64 +resource-server.build.arm64: + @mkdir -p docker/dist/arm64 + @cp docker/service/resource-server.Dockerfile docker/dist/arm64/resource-server.Dockerfile + @sed -i -e '/#nginx.Dockerfile/ {' -e 'r docker/base/arm64/nginx.Dockerfile' -e 'd' -e '}' docker/dist/arm64/resource-server.Dockerfile + @CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -v -o docker/dist/reaper cmd/reaper/main.go + @docker build -f docker/dist/arm64/resource-server.Dockerfile --tag ${MAKE_IMAGE} . + +resource-server.upload.arm64: MAKE_IMAGE ?= ${IMAGE_REPOSITORY}/resource-server:${VERSION}-arm64 +resource-server.upload.arm64: resource-server.build.arm64 + @docker push ${MAKE_IMAGE} + +pre-build: + @mkdir -p bin/ + +build-zgctl-all-platforms: pre-build + $(foreach GOOS, $(PLATFORMS),\ + $(foreach GOARCH, $(ARCHITECTURES), $(shell export GOOS=$(GOOS); export GOARCH=$(GOARCH); CGO_ENABLED=0 go build -v -o bin/zgctl-$(GOOS)-$(GOARCH) cmd/zgctl/main.go))) + +zgctl-sidecar.build.amd64: MAKE_IMAGE ?= ${IMAGE_REPOSITORY}/zgctl-sidecar:${VERSION}-amd64 +zgctl-sidecar.build.amd64: + @docker build -f docker/service/syncthing.Dockerfile --tag ${MAKE_IMAGE} . + +zgctl-sidecar.upload.amd64: MAKE_IMAGE ?= ${IMAGE_REPOSITORY}/zgctl-sidecar:${VERSION}-amd64 +zgctl-sidecar.upload.amd64: zgctl-sidecar.build.amd64 + @docker push ${MAKE_IMAGE} + +zadig-debug.build.amd64: MAKE_IMAGE ?= ${IMAGE_REPOSITORY}/zadig-debug:${VERSION}-amd64 +zadig-debug.build.amd64: + @docker build -f docker/service/zadig-debug.Dockerfile --tag ${MAKE_IMAGE} . + +zadig-debug.upload.amd64: MAKE_IMAGE ?= ${IMAGE_REPOSITORY}/zadig-debug:${VERSION}-amd64 +zadig-debug.upload.amd64: zadig-debug.build.amd64 + @docker push ${MAKE_IMAGE} + .PHONY: clean clean: @rm -rf docker/dist + @rm -rf bin/ diff --git a/README-zh-CN.md b/README-zh-CN.md index ecfe7fd76a2d3bef96a13739634b84e6a1c02f28..22566c74e6c698cf11f66ea0f68ad31174198ce6 100644 --- a/README-zh-CN.md +++ b/README-zh-CN.md @@ -33,7 +33,7 @@ ## Zadig 介绍 -Zadig 是一款面向开发者设计的云原生持续交付(Continuous Delivery)产品，具备高可用 CI/CD 能力，提供云原生运行环境，支持开发者本地联调、微服务并行构建和部署、集成测试等。Zadig 不改变现有流程，无缝集成 Github/Gitlab、Jenkins、多家云厂商等，运维成本极低。我们的目标是通过云原生技术的运用和工程产品赋能，打造极致、高效、愉悦的开发者工作体验，让工程师成为企业创新的核心引擎。 +Zadig 是 KodeRover 公司基于 Kubernetes 自主设计、研发的开源分布式持续交付 (Continuous Delivery) 产品，具备灵活易用的高并发工作流、面向开发者的云原生环境、高效协同的测试管理、强大免运维的模板库、客观精确的效能洞察以及云原生 IDE 插件等重要特性，为工程师提供统一的协作平面。Zadig 内置了 K8s YAML、Helm Chart、主机等复杂场景最佳实践，适用大规模微服务、高频高质量交付等场景。我们的目标是通过云原生技术的运用和工程产品赋能，打造极致、高效、愉悦的开发者工作体验，让工程师成为企业创新的核心引擎。 我们的愿景：`工程师 + Zadig = 商业上的成功` @@ -42,33 +42,49 @@ Zadig 是一款面向开发者设计的云原生持续交付(Continuous Delivery ![业务架构图](./Zadig-Business-Architecture-zh.jpg) -产品特性介绍： - -- **高并发的工作流** - -基于云原生设计，经过简单配置，系统自动生成工作流，实现多服务高并发执行构建部署测试任务，以解决微服务架构下带来的多服务构建部署效率低下问题。 - -- **以服务为核心的集成环境** +想了解更多系统架构信息，参考 [系统架构简介](System-Architecture-Overview-zh-CN.md). -一套服务配置，分钟级创建多套数据隔离的测试环境。为开发者进行日常调试、为测试人员做集成测试、为产品经理对外 Demo 提供强力支撑。 +产品特性介绍： -对于现有的环境无需担心迁移成本，一键托管，轻松浏览、调试环境中的所有服务。 +
+ 灵活易用的高并发工作流 + 简单配置，可自动生成高并发工作流，多个微服务可并行构建、并行部署、并行测试，大大提升代码验证效率。自定义的工作流步骤，配合人工审核，灵活且可控的保障业务交付质量。 +
-- **无侵入的自动化测试** +
+ 面向开发者的云原生环境 + 分钟级创建或复制一套完整的隔离环境，应对频繁的业务变更和产品迭代。基于全量基准环境，快速为开发者提供一套独立的自测环境。一键托管集群资源即可轻松调试已有服务，验证业务代码。 +
-便捷且无侵入的对接已有自动化测试框架，通过 GitHub/GitLab Webhook 自动构建、部署及测试。 +
+ 高效协同的测试管理 + 便捷对接 Jmeter、Pytest 等主流测试框架，跨项目管理和沉淀 UI、API、E2E 测试用例资产。通过工作流，向开发者提供前置测试验证能力。通过持续测试和质量分析，充分释放测试价值。 +
-通过办公通讯机器人为开发者提供第一时间质量反馈，精准高效。有效落地“测试左移”工程实践，让测试价值得到体现。 +
+ 强大免运维的模板库 + 跨项目共享 K8s YAML 模版、Helm Chart 模版、构建模版等，实现配置的统一化管理。基于一套模板可创建数百微服务，开发工程师少量配置可自助使用，大幅降低运维管理负担。 +
-- **开发本地联调 CLI** +
+ 客观精确的效能洞察 + 全面了解系统运行状态，包括集群、项目、环境、工作流，关键过程通过率等数据概览。提供项目维度的构建、测试、部署等客观的效能度量数据，精准分析研发效能短板，促进稳步提升。 +
-开发本地编辑完代码，一键进行本地代码构建，部署到联调环境，无需再陷入复杂且繁琐的工作流程，让本地联调不再成为难事。解放工程师双手，去创造更多产品价值。 +
+ 云原生 IDE 插件 + 开发者无需平台切换，在 VScode IDE 中即可获得 Zadig 产品核心能力。编写代码后，无需打包镜像，即可一键热部署到自测环境，快速完成自测、联调和集成验证，开发效率倍增。 +
## 快速上手 ### 快速使用 -请参阅 [快速入门](https://docs.koderover.com/zadig/quick-start/try-out-install) +请参阅 [快速入门](https://docs.koderover.com/zadig/quick-start/try-out-install/) + +### 训练营 + +Zadig [训练营](https://github.com/koderover/zadig-bootcamp)主要是为开发者提供实践小技巧、最佳实践案例的搭建、典型应用场景的演示等，以便快速获得持续交付最佳解决方案。可以直接进入 [教程](https://www.koderover.com/tutorials) 一步步实践和尝试。 ### 快速开发 diff --git a/README.md b/README.md index 92c3a999b1944f03e09c86efd9062485062857c2..b004e68e76c800a72f214f1606f5aeb39af10b26 100644 --- a/README.md +++ b/README.md @@ -41,37 +41,50 @@ Zadig is non-invasive, it does not exclude any of your existing development proc > Our vision is: Developer + Zadig = Business success -The architecture is as follows: +The business architecture is as follows: ![Business Architecture](./Zadig-Business-Architecture.jpg) +For more details about system architecture, see [Zadig System Architecture Overview](System-Architecture-Overview.md). + The Highlighted Features: -- **High Concurrency** +
+ High Concurrency + Based on cloud-native design, through simple configuration, the system automatically generates workflows to achieve high concurrent execution for continuous delivery relevant tasks such as building, testing and deployment, across multiple services. It significantly improves the efficiency of multi-services deployment in microservice architecture. +
-Based on cloud-native design, through simple configuration, the system automatically generates workflows to achieve high concurrent execution for continuous delivery relevant tasks such as building, testing and deployment, across multiple services. It significantly improves the efficiency of multi-services deployment in microservice architecture. +
+ Service-oriented Environment + With just one set of service configuration, multiple encapsulated environments will be provided automatically within minutes, empowering independent environments for developers, QAs and product managers. -- **Service-oriented Environment** + Minimum to none migration cost of existing environments -- just hosting with one click, the system allows browsing and adjusting all the services at your fingertips. +
-With just one set of service configuration, multiple encapsulated environments will be provided automatically within minutes, empowering independent environments for developers, QAs and product managers. +
+ Non-intrusive Testing Automation + Zadig can easily and non-intrusively embed existing testing automation frameworks, and achieve continuous building, testing and deployment via GitHub/GitLab Webhook. -Minimum to none migration cost of existing environments -- just hosting with one click, the system allows browsing and adjusting all the services at your fingertips. + It also integrates with productivity bots to provide instant quality report, which effectively applies shift-left testing best practices. +
-- **Non-intrusive Testing Automation** +
+ Convenient Development CLI/IDE Plugin + Zadig also provides a convenient toolkit with development commandline interface which allows compiling, building and deploying the changes to dev environment with one command. It enables collaborated debugging and testing with minimum manual toil, reduces cognitive load and allows teams to focus more on business. +
-Zadig can easily and non-intrusively embed existing testing automation frameworks, and achieve continuous building, testing and deployment via GitHub/GitLab Webhook. -It also integrates with productivity bots to provide instant quality report, which effectively applies shift-left testing best practices. +## Quick start -- **Convenient Development CLI** +### How to use? -Zadig also provides a convenient toolkit with development commandline interface which allows compiling, building and deploying the changes to dev environment with one command. It enables collaborated debugging and testing with minimum manual toil, reduces cognitive load and allows teams to focus more on business. +Please follow [Quick Start](https://docs.koderover.com/zadig/quick-start/try-out-install/) -## Quick start +### Bootcamps -### How to use? +With many hands-on tips, case studies and demos of different application types, Zadig [bootcamp](https://github.com/koderover/zadig-bootcamp) gives developers easy to follow best practices in a cloud native CI/CD solution -Please follow [Quick Start](https://docs.koderover.com/zadig/quick-start/try-out-install) +Just follow [Tutorials](https://www.koderover.com/tutorials) to try it step by step. ### How to make contribution? diff --git a/System-Architecture-Overview-zh-CN.md b/System-Architecture-Overview-zh-CN.md new file mode 100644 index 0000000000000000000000000000000000000000..dcef018dff66cc5e84a1740d54990954730177f1 --- /dev/null +++ b/System-Architecture-Overview-zh-CN.md @@ -0,0 +1,39 @@ +# Zadig 系统架构简介 + +## 系统架构图 + +![Architecture_diagram](./Zadig-System-Architecture.svg) + +## 核心组件介绍 + +用户入口： +- zadig-portal：Zadig 前端组件 +- kodespace：Zadig 开发者命令行工具 +- Zadig Toolkit：vscode 开发者插件 + +API 网关： +- [Gloo Edge](https://github.com/solo-io/gloo)：Zadig 的 API 网关组件 +- [OPA](https://github.com/open-policy-agent/opa)：认证和授权组件 +- [Dex](https://github.com/dexidp/dex)：Zadig 的身份认证服务，用于连接其他第三方认证系统，比如 AD / LDAP / OAuth2 / GitHub / .. +- User：用户管理，Token 生成 + +Zadig 核心业务： +- Picket：数据聚合服务 +- Aslan：项目 / 环境 / 服务 / 工作流 / 构建配置 / 系统配置等系统功能 +- Policy：OPA 数据源，策略注册中心 +- Config：系统配置 +- Workflow Runner： + - warpdrive：工作流引擎，负责 reaper、predator 实例的创建销毁等管理操作 + - reaper：负责执行单个工作流作业中的构建、测试等任务 + - predator：负责执行单个工作流作业中的镜像分发任务 + - plugins：工作流插件 + - Jenkins-plugin：用于触发 Jenkins job，显示状态和结果等 +- Cron：定时任务，包括环境的回收，K8s 资源的清理等 +- NSQ：消息队列（第三方组件） + +数据平面： +- MongoDB：业务数据数据库 +- MySQL：存储 dex 配置、用户信息的数据库 + +K8s 集群： +- Zadig 业务运行在各种云厂商的标准 K8s 集群 \ No newline at end of file diff --git a/System-Architecture-Overview.md b/System-Architecture-Overview.md new file mode 100644 index 0000000000000000000000000000000000000000..9c9414992911e9a950c1b7af560cf2d3b4c5779d --- /dev/null +++ b/System-Architecture-Overview.md @@ -0,0 +1,39 @@ +# Zadig System Architecture Overview + +## System Architecture + +![Architecture_diagram](./Zadig-System-Architecture.svg) + +## Main Components + +User Interface： +- zadig-portal：Zadig web component +- kodespace：Zadig command line tools +- Zadig Toolkit：vscode plugin + +API Gateway： +- [Gloo Edge](https://github.com/solo-io/gloo)：Zadig API gateway +- [OPA](https://github.com/open-policy-agent/opa)：Authentication and authorization +- [Dex](https://github.com/dexidp/dex)：Identity service for Zadig, acts as a portal to other identity providers like AD / LDAP / OAuth2 / GitHub / .. +- User：User management, token generation + +Zadig Core： +- Picket：backend for frontend service. +- Aslan：main service for all business logic. Project, environment, service, workflow, build, system management are all in this service. +- Policy：data source of OPA and policy registration center. +- Config：system configuration center. +- Workflow Runner： + - warpdrive：workflow engine, manages reaper and predator + - reaper：workflow runner. Used for building, testing tasks. + - predator：workflow runner. Used for distribute tasks. + - plugins：workflow plugins + - Jenkins-plugin：workflow runner. Used as connector to trigger Jenkins job and retrieve job information. +- Cron：cronjob runner +- NSQ：message queue + +Data Plane： +- MongoDB：database for business data. +- MySQL：dex configuration,database for user. + +Kubernetes Cluster： +- Zadig business runs on standard K8s clusters from cloud vendors diff --git a/Zadig-Business-Architecture-zh.jpg b/Zadig-Business-Architecture-zh.jpg index a7edb8a09cec859d78ad097d9cb9f0e338c48b53..bca27de195439cd326197f0d8768082ab9b2102c 100644 Binary files a/Zadig-Business-Architecture-zh.jpg and b/Zadig-Business-Architecture-zh.jpg differ diff --git a/Zadig-Business-Architecture.jpg b/Zadig-Business-Architecture.jpg index a962edd30d1a60cf8b0a9c7a7972677be37a8a69..6140e9b3bb712901ebe4cbfef2e61cce533c4745 100644 Binary files a/Zadig-Business-Architecture.jpg and b/Zadig-Business-Architecture.jpg differ diff --git a/Zadig-System-Architecture.svg b/Zadig-System-Architecture.svg new file mode 100644 index 0000000000000000000000000000000000000000..c53a8eacdabc009d1c555c0a6de3c201d713d740 --- /dev/null +++ b/Zadig-System-Architecture.svg @@ -0,0 +1 @@ +
User
Interface
User...
API
Gateway
API...
OAuth 2.0
OAuth 2.0
LDAP/AD
LDAP/AD
GitHub
GitHub
……
……
OpenID Provider
OpenID Provider
Zadig Core
Zadig...
Data Plane
Data Plane
MongoDB
MongoDB
MySQL
MySQL
Engineer (development, testing, SRE etc.)
Engineer (development, testing, SRE etc.)
Workflow Runner
Workflow Runner
warpdrive
warpdrive
Dind Replicas
Dind Replicas
dind
dind
dind
dind
NSQ
NSQ
Cron
Cron
workflow controller
workflow controller
project
project
service
service
environment
environment
cronjob controller
cronjob controller
Zadig Portal
Zadig Portal
Zadig Toolkit
Zadig Toolkit
Kodespace
Kodespace
reaper
reaper
predator
predat...
plugins
plugins
warpdrive
warpdr...
warpdrive
warpdr...
Picket
Picket
Aslan
Aslan
Dex
Dex
User
User
Gloo Edge
Gloo Edge
Opa
Opa
Kubernetes
Cluster
Kubern...
Ali ACK 
Ali ACK 
AWS EKS
AWS EKS
Tencent TKE
Tencent TKE
Huawei CCE
Huawei CCE
......
......
Config
Config
Policy
PolicyViewer does not support full SVG 1.1 \ No newline at end of file diff --git a/action.yml b/action.yml index 16e71a8d1dd33264bc7bdfbaea98da014868c339..a3b84c3c8802eb4ceabb29ed46f110093a9f4d94 100644 --- a/action.yml +++ b/action.yml @@ -1,3 +1,5 @@ +# webhook test +# webhook test name: 'Labeler' description: 'Automatically label new pull requests based on the paths of files being changed' author: 'GitHub' @@ -15,4 +17,4 @@ inputs: runs: using: 'node12' - main: 'dist/index.js' + main: 'dist/index.js' \ No newline at end of file diff --git a/cmd/aslan/main.go b/cmd/aslan/main.go index 6c4e7a98ffb7f6a51f2453386bb3a3dfde5db1c0..38956e3d7d123798e30751a31d020493467e0d05 100644 --- a/cmd/aslan/main.go +++ b/cmd/aslan/main.go @@ -34,5 +34,22 @@ func main() { if err := server.Serve(ctx); err != nil { log.Fatal(err) + log.Fatal(err) + log.Fatal(err) + log.Fatal(err) + log.Fatal(err) + log.Fatal(err) + log.Fatal(err) + log.Fatal(err) + log.Fatal(err) + log.Fatal(err) + log.Fatal(err) + log.Fatal(err) + log.Fatal(err) + log.Fatal(err) + log.Fatal(err) + log.Fatal(err) + log.Fatal(err) + log.Fatal(err) } } diff --git a/cmd/config/main.go b/cmd/config/main.go new file mode 100644 index 0000000000000000000000000000000000000000..3df10041fcaf3373bf0151575fc74641422e7e57 --- /dev/null +++ b/cmd/config/main.go @@ -0,0 +1,50 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "flag" + "log" + "os/signal" + "syscall" + + "github.com/spf13/pflag" + "github.com/spf13/viper" + + "github.com/koderover/zadig/pkg/microservice/systemconfig/config" + "github.com/koderover/zadig/pkg/microservice/systemconfig/server" +) + +func main() { + + flag.String(config.FeatureFlag, "", "turn a set of features on or off") + + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) + pflag.Parse() + viper.BindPFlags(pflag.CommandLine) + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + go func() { + <-ctx.Done() + stop() + }() + + if err := server.Serve(ctx); err != nil { + log.Fatal(err) + } +} diff --git a/cmd/hub-agent/main.go b/cmd/hub-agent/main.go new file mode 100644 index 0000000000000000000000000000000000000000..c42fe3a87b08ca170173763b22565bd2c6856d79 --- /dev/null +++ b/cmd/hub-agent/main.go @@ -0,0 +1,38 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "log" + "os/signal" + "syscall" + + "github.com/koderover/zadig/pkg/microservice/hubagent/server" +) + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + go func() { + <-ctx.Done() + stop() + }() + + if err := server.Serve(ctx); err != nil { + log.Fatal(err) + } +} diff --git a/cmd/hubserver/main.go b/cmd/hub-server/main.go similarity index 100% rename from cmd/hubserver/main.go rename to cmd/hub-server/main.go diff --git a/cmd/hubagent/main.go b/cmd/init/main.go similarity index 82% rename from cmd/hubagent/main.go rename to cmd/init/main.go index 8e7a30db49bc5ac6bcee54c4a1ba7370a118bba0..6d10e2d038b6eafc5d298e0a966edc2df7944f25 100644 --- a/cmd/hubagent/main.go +++ b/cmd/init/main.go @@ -17,13 +17,9 @@ limitations under the License. package main import ( - "log" - - "github.com/koderover/zadig/pkg/microservice/hubagent/server" + "github.com/koderover/zadig/pkg/cli/initconfig/cmd" ) func main() { - if err := server.Serve(); err != nil { - log.Fatal(err) - } + cmd.Execute() } diff --git a/cmd/jenkinsplugin/main.go b/cmd/jenkins-plugin/main.go similarity index 100% rename from cmd/jenkinsplugin/main.go rename to cmd/jenkins-plugin/main.go diff --git a/cmd/jobexecutor/main.go b/cmd/jobexecutor/main.go new file mode 100644 index 0000000000000000000000000000000000000000..d231bd90db40b3b772655f0969df0f0194332474 --- /dev/null +++ b/cmd/jobexecutor/main.go @@ -0,0 +1,38 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "log" + "os/signal" + "syscall" + + "github.com/koderover/zadig/pkg/microservice/jobexecutor/executor" +) + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + go func() { + <-ctx.Done() + stop() + }() + + if err := executor.Execute(ctx); err != nil { + log.Fatal(err) + } +} diff --git a/cmd/packager-plugin/main.go b/cmd/packager-plugin/main.go new file mode 100644 index 0000000000000000000000000000000000000000..bd2fa8bc1bbdca5db6e0112706415a8c2985d95a --- /dev/null +++ b/cmd/packager-plugin/main.go @@ -0,0 +1,29 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "log" + + "github.com/koderover/zadig/pkg/microservice/packager/executor" +) + +func main() { + if err := executor.Execute(); err != nil { + log.Fatalf("Failed to run packager, the error is: %+v", err) + } +} diff --git a/cmd/predator/main.go b/cmd/predator-plugin/main.go similarity index 100% rename from cmd/predator/main.go rename to cmd/predator-plugin/main.go diff --git a/cmd/upgradeassistant/main.go b/cmd/ua/main.go similarity index 100% rename from cmd/upgradeassistant/main.go rename to cmd/ua/main.go diff --git a/pkg/tool/kube/getter/role.go b/cmd/user/main.go similarity index 64% rename from pkg/tool/kube/getter/role.go rename to cmd/user/main.go index 56e53469bf604767729aa68dc9bc38aa15b784cd..2f8f7c699280caa206d914658a629e04c980188d 100644 --- a/pkg/tool/kube/getter/role.go +++ b/cmd/user/main.go @@ -14,19 +14,25 @@ See the License for the specific language governing permissions and limitations under the License. */ -package getter +package main import ( - rbacv1beta1 "k8s.io/api/rbac/v1beta1" - "sigs.k8s.io/controller-runtime/pkg/client" + "context" + "log" + "os/signal" + "syscall" + + "github.com/koderover/zadig/pkg/microservice/user/server" ) -func GetRole(ns, name string, cl client.Client) (*rbacv1beta1.Role, bool, error) { - g := &rbacv1beta1.Role{} - found, err := GetResourceInCache(ns, name, g, cl) - if err != nil || !found { - g = nil - } +func main() { + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + go func() { + <-ctx.Done() + stop() + }() - return g, found, err + if err := server.Serve(ctx); err != nil { + log.Fatal(err) + } } diff --git a/cmd/zgctl/cmd/root.go b/cmd/zgctl/cmd/root.go new file mode 100644 index 0000000000000000000000000000000000000000..736613dbd578dff84e315e95fa1e74509cfaa25f --- /dev/null +++ b/cmd/zgctl/cmd/root.go @@ -0,0 +1,59 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "os" + "path/filepath" + + "github.com/spf13/cobra" + + "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/log" +) + +var zadigHost string +var zadigToken string +var homeDir string + +func init() { + rootCmd.PersistentFlags().StringVar(&zadigHost, "zadig-host", "", "host of zadig system") + rootCmd.PersistentFlags().StringVar(&zadigToken, "zadig-token", "", "token to access zadig system") + rootCmd.PersistentFlags().StringVar(&homeDir, "home-dir", "$HOME/.zadig", "home dir of plugin") + + log.Init(&log.Config{ + Level: config.LogLevel(), + SendToFile: config.SendLogToFile(), + Filename: filepath.Join(os.ExpandEnv(homeDir), "log"), + Development: config.Mode() != setting.ReleaseMode, + MaxSize: 5, + }) +} + +var rootCmd = &cobra.Command{ + Use: "zgctl", + Short: "zgctl is a cli used for IDE plugin.", + RunE: func(cmd *cobra.Command, args []string) error { + + return nil + }, +} + +func Execute() error { + return rootCmd.Execute() +} diff --git a/cmd/zgctl/cmd/serve.go b/cmd/zgctl/cmd/serve.go new file mode 100644 index 0000000000000000000000000000000000000000..54efa8e7f0969e4ec8e8cd17e37d527feaab1945 --- /dev/null +++ b/cmd/zgctl/cmd/serve.go @@ -0,0 +1,76 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/dgraph-io/badger/v3" + "github.com/gin-gonic/gin" + "github.com/spf13/cobra" + + "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/zgctl" + ginmiddleware "github.com/koderover/zadig/pkg/middleware/gin" + "github.com/koderover/zadig/pkg/tool/log" +) + +var listenAddr string + +func init() { + serveCmd.Flags().StringVar(&listenAddr, "listen-addr", ":8080", "listen address of zgctl serve") + + rootCmd.AddCommand(serveCmd) +} + +var serveCmd = &cobra.Command{ + Use: "serve", + Short: "provide APIs for IDE Plugin", + Long: "serve is a sub cmd used to provide APIs for IDE Plugin.", + RunE: func(cmd *cobra.Command, args []string) error { + r := gin.Default() + injectMiddlewares(r) + + homeDir := os.ExpandEnv(homeDir) + dbPath := filepath.Join(homeDir, "db") + db, err := badger.Open(badger.DefaultOptions(dbPath)) + if err != nil { + return fmt.Errorf("failed to open db %q: %s", dbPath, err) + } + defer db.Close() + + apiv1 := r.Group("/api/v1") + router := zgctl.NewRouter(&zgctl.RouterConfig{ + ZadigHost: zadigHost, + ZadigToken: zadigToken, + HomeDir: homeDir, + DB: db, + }) + router.Inject(apiv1) + + return r.Run(listenAddr) + }, +} + +func injectMiddlewares(g *gin.Engine) { + g.Use(ginmiddleware.Response()) + g.Use(ginmiddleware.RequestID()) + g.Use(ginmiddleware.RequestLog(log.NewFileLogger(config.RequestLogFile()))) + g.Use(gin.Recovery()) +} diff --git a/cmd/zgctl/main.go b/cmd/zgctl/main.go new file mode 100644 index 0000000000000000000000000000000000000000..baeb50cc530e7a3a26dd0b2b993e84c892152e9c --- /dev/null +++ b/cmd/zgctl/main.go @@ -0,0 +1,25 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/koderover/zadig/cmd/zgctl/cmd" +) + +func main() { + cmd.Execute() +} diff --git a/community/dev/contributor-workflow.md b/community/dev/contributor-workflow.md index e7289a5a90bff9ace3396a8a4b2f2493f3692aa1..868a58d3913a8f9e807f8fb3f6c3a01b45360ba5 100644 --- a/community/dev/contributor-workflow.md +++ b/community/dev/contributor-workflow.md @@ -1,20 +1,16 @@ # Zadig 开发流程 -## 目录 - -- [Zadig 开发流程](#zadig-开发流程) - - [目录](#目录) - - [1. 克隆代码](#1-克隆代码) - - [2. 本地开发环境搭建](#2-本地开发环境搭建) - - [前端环境](#前端环境) - - [后端环境](#后端环境) - - [3. 贡献代码](#3-贡献代码) - - [4. 调试](#4-调试) - - [方式一：云上代码测试](#方式一云上代码测试) - - [方式二：本地代码测试](#方式二本地代码测试) - - [前端本地测试](#前端本地测试) - - [后端本地测试](#后端本地测试) +- [1. 克隆代码](#1-克隆代码) +- [2. 本地开发环境搭建](#2-本地开发环境搭建) + - [前端环境](#前端环境) + - [后端环境](#后端环境) +- [3. 贡献代码](#3-贡献代码) +- [4. 调试](#4-调试) + - [方式一：云上代码测试](#方式一云上代码测试) + - [方式二：本地代码测试](#方式二本地代码测试) + - [前端本地测试](#前端本地测试) + - [后端本地测试](#后端本地测试) ## 1. 克隆代码 @@ -49,13 +45,27 @@ Zadig 为您提供云上测试环境 [https://os.koderover.com](https://os.koder 第 1 步：GitHub OAuth 登录系统 -第 2 步：Fork Zadig 项目 +![Login through Github](./github_login.png) -![Fork project](./fork-zadig-option.png) +第 2 步：在 Zadig 项目中获得开发者工作流和环境 -在弹出的窗口中，点击 zadig 选项，将其中的 endpoint.FQDN 中的 `githubid` 改成您的 GitHub ID，否则可能无法访问。 +步骤如下： +1. 点击 Zadig 项目 +2. 在页面中点击 环境变量 按钮，如下图所示 -![Update githubid](./fork-zadig-vars.png) +![Edit variable button](./edit_variable_button.png) +3. 在弹出的窗口中，如图修改如下字段： + + 1. dex.config.issuer=http://${github_id}-dex:5556/dex + 2. dex.config.staticClients[0].redirectURISs[0]=http://${github_id}.ko.coderover.cn/api/v1/callback + 3. dex.fullnameOverride=${github_id}-dex + 4. endpoint.FQDN=${github_id}.ko.coderover.cn + 5. global.extensions.extAuth.extauthzServerRef.namespace=zadig-env-${环境名称} + +![Fork_varibales_1](./fork_variables_1.png) +![Fork_varibales_2](./fork_variables_2.png) + +请注意，如果没有按照以上规则修改参数，可能会导致环境无法创建/创建环境后无法正常访问！ Fork 完成后，您将获得一个 Zadig 测试环境。 @@ -90,4 +100,4 @@ yarn run dev #### 后端本地测试 -使用 Zadig CLI 进行本地调试，使用方式参见 [Zadig CLI 使用指南](https://docs.koderover.com/zadig/cli/kodespace-usage-for-contributor) +使用 Zadig CLI 进行本地调试，使用方式参见 [Zadig CLI 使用指南](https://docs.koderover.com/zadig/v1.7.1/cli/kodespace-usage-for-contributor/) diff --git a/community/dev/edit_variable_button.png b/community/dev/edit_variable_button.png new file mode 100644 index 0000000000000000000000000000000000000000..3d31e367d14f7bb6837f921bae740c40969676c1 Binary files /dev/null and b/community/dev/edit_variable_button.png differ diff --git a/community/dev/fork-zadig-option.png b/community/dev/fork-zadig-option.png deleted file mode 100644 index 5b4125cc3c5ece845394c6ca38ab34ed32b85236..0000000000000000000000000000000000000000 Binary files a/community/dev/fork-zadig-option.png and /dev/null differ diff --git a/community/dev/fork-zadig-vars.png b/community/dev/fork-zadig-vars.png deleted file mode 100644 index 979ea70779d63331a6e90cd51b91c25214e8fc74..0000000000000000000000000000000000000000 Binary files a/community/dev/fork-zadig-vars.png and /dev/null differ diff --git a/community/dev/fork_variables_1.png b/community/dev/fork_variables_1.png new file mode 100644 index 0000000000000000000000000000000000000000..1097b45d9048117a29431f09106a068ff0e56b78 Binary files /dev/null and b/community/dev/fork_variables_1.png differ diff --git a/community/dev/fork_variables_2.png b/community/dev/fork_variables_2.png new file mode 100644 index 0000000000000000000000000000000000000000..16d5be36b8e4679ca5c192c39b8bc9cb734b6c91 Binary files /dev/null and b/community/dev/fork_variables_2.png differ diff --git a/community/dev/github_login.png b/community/dev/github_login.png new file mode 100644 index 0000000000000000000000000000000000000000..ab1c59170744146eef9c01d4879b92524ed4040e Binary files /dev/null and b/community/dev/github_login.png differ diff --git a/docker/base/amd64/alpine-curl.Dockerfile b/docker/base/amd64/alpine-curl.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..f08056e83da7d091dab99f269513b00f09ce4584 --- /dev/null +++ b/docker/base/amd64/alpine-curl.Dockerfile @@ -0,0 +1,8 @@ +FROM alpine/curl:3.14 + +# https://wiki.alpinelinux.org/wiki/Setting_the_timezone +RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories && \ + apk add tzdata && \ + cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ + echo Asia/Shanghai > /etc/timezone && \ + apk del tzdata diff --git a/docker/base/amd64/ubuntu-bionic.Dockerfile b/docker/base/amd64/ubuntu-bionic.Dockerfile index 80043633f5b89fa0f2f20db339e1cfb041f4a0d7..87608d6bf2122f8ef84809ff8be4c0e06613c3cf 100644 --- a/docker/base/amd64/ubuntu-bionic.Dockerfile +++ b/docker/base/amd64/ubuntu-bionic.Dockerfile @@ -1 +1 @@ -FROM ubuntu:bionic +FROM koderover.tencentcloudcr.com/koderover-public/build-base:bionic-amd64 diff --git a/docker/base/amd64/ubuntu-focal.Dockerfile b/docker/base/amd64/ubuntu-focal.Dockerfile index 9cc9f1a394016922969759278abae9f238e9db32..32eaf34bd54294bc603359ec4228774ac284ba99 100644 --- a/docker/base/amd64/ubuntu-focal.Dockerfile +++ b/docker/base/amd64/ubuntu-focal.Dockerfile @@ -1 +1 @@ -FROM ubuntu:focal +FROM koderover.tencentcloudcr.com/koderover-public/build-base:focal-amd64 diff --git a/docker/base/amd64/ubuntu-xenial.Dockerfile b/docker/base/amd64/ubuntu-xenial.Dockerfile index ccb4277ff2f4e52d4d9e2d2e1054d192bb5126e8..05ba3257eb25018a6fa49f2ad94ec1c6502d9c5e 100644 --- a/docker/base/amd64/ubuntu-xenial.Dockerfile +++ b/docker/base/amd64/ubuntu-xenial.Dockerfile @@ -1,16 +1 @@ -FROM ubuntu:xenial - -# 修改镜像源和时区 -RUN sed -i -E "s/[a-zA-Z0-9]+.ubuntu.com/mirrors.aliyun.com/g" /etc/apt/sources.list \ - && apt-get clean && apt-get update && apt-get install -y apt-transport-https ca-certificates \ - && apt-get install -y \ - tzdata \ - net-tools \ - dnsutils \ - ca-certificates \ - git \ - curl \ - lsof \ - telnet \ - && ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \ - && rm -rf /var/lib/apt/lists/* +FROM koderover.tencentcloudcr.com/koderover-public/build-base:xenial-amd64 diff --git a/docker/base/arm64/alpine-curl.Dockerfile b/docker/base/arm64/alpine-curl.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..3bda0ad84e1fccfb5d0770109fb3167041ad7266 --- /dev/null +++ b/docker/base/arm64/alpine-curl.Dockerfile @@ -0,0 +1,11 @@ +FROM arm64v8/alpine:3.15.2 + +RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories && \ + apk add curl + +# https://wiki.alpinelinux.org/wiki/Setting_the_timezone +RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories && \ + apk add tzdata && \ + cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ + echo Asia/Shanghai > /etc/timezone && \ + apk del tzdata diff --git a/docker/base/arm64/alpine-git.Dockerfile b/docker/base/arm64/alpine-git.Dockerfile index a72261a4962ffd52425294511d49e4b1fd20d93d..51fb7450dfbc746997d8c2d7304b03b2f22fe8aa 100644 --- a/docker/base/arm64/alpine-git.Dockerfile +++ b/docker/base/arm64/alpine-git.Dockerfile @@ -1,4 +1,4 @@ -FROM arm64v8/alpine +FROM arm64v8/alpine:3.15.2 RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories && \ apk add git less openssh diff --git a/docker/base/arm64/alpine.Dockerfile b/docker/base/arm64/alpine.Dockerfile index 1acc539a0169410cceddffea99d1fe7a9a51ed5b..3e2d556d7a083745b06b28af1fb746dbd748011e 100644 --- a/docker/base/arm64/alpine.Dockerfile +++ b/docker/base/arm64/alpine.Dockerfile @@ -1,4 +1,4 @@ -FROM arm64v8/alpine +FROM arm64v8/alpine:3.15.2 # https://wiki.alpinelinux.org/wiki/Setting_the_timezone RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories && \ diff --git a/docker/base/arm64/ubuntu-bionic.Dockerfile b/docker/base/arm64/ubuntu-bionic.Dockerfile index 9760924741c71445377d59c982aec0119f2a6065..ffeeda2e774aef140ad91a3c3305444d5cb738e2 100644 --- a/docker/base/arm64/ubuntu-bionic.Dockerfile +++ b/docker/base/arm64/ubuntu-bionic.Dockerfile @@ -1 +1 @@ -FROM arm64v8/ubuntu:bionic +FROM koderover.tencentcloudcr.com/koderover-public/build-base:bionic-arm64 diff --git a/docker/base/arm64/ubuntu-focal.Dockerfile b/docker/base/arm64/ubuntu-focal.Dockerfile index e416d2435cf33d9b60f828ea980d454385c9fbbf..0b9385e68d8a2cf1da4648758aa95b6d83a6b331 100644 --- a/docker/base/arm64/ubuntu-focal.Dockerfile +++ b/docker/base/arm64/ubuntu-focal.Dockerfile @@ -1 +1 @@ -FROM arm64v8/ubuntu:focal +FROM koderover.tencentcloudcr.com/koderover-public/build-base:focal-arm64 diff --git a/docker/base/arm64/ubuntu-xenial.Dockerfile b/docker/base/arm64/ubuntu-xenial.Dockerfile index e2f37c5fb752440c49ae0c2235f1bfa0ea4a6779..75bb2bd861f946d66dd9c0ab888518633b0ec4f5 100644 --- a/docker/base/arm64/ubuntu-xenial.Dockerfile +++ b/docker/base/arm64/ubuntu-xenial.Dockerfile @@ -1,16 +1,2 @@ -FROM arm64v8/ubuntu:xenial +FROM koderover.tencentcloudcr.com/koderover-public/build-base:xenial-arm64 -# 修改镜像源和时区 -RUN sed -i -E "s/[a-zA-Z0-9]+.ubuntu.com/mirrors.aliyun.com/g" /etc/apt/sources.list \ - && apt-get clean && apt-get update && apt-get install -y apt-transport-https ca-certificates \ - && apt-get install -y \ - tzdata \ - net-tools \ - dnsutils \ - ca-certificates \ - git \ - curl \ - lsof \ - telnet \ - && ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \ - && rm -rf /var/lib/apt/lists/* diff --git a/docker/service/reaper-plugin-focal.Dockerfile b/docker/base/image/bionic-base-amd64.Dockerfile similarity index 77% rename from docker/service/reaper-plugin-focal.Dockerfile rename to docker/base/image/bionic-base-amd64.Dockerfile index ed2d6abf22fe2cd44f3bc780ae398739ef7efd1b..3f7b22ca5f7c12fac067f9f7cd66c47d391b1ef4 100644 --- a/docker/service/reaper-plugin-focal.Dockerfile +++ b/docker/base/image/bionic-base-amd64.Dockerfile @@ -1,8 +1,4 @@ -#golang.Dockerfile - -RUN go build -v -o /reaper ./cmd/reaper/main.go - -#ubuntu-focal.Dockerfile +FROM ubuntu:bionic RUN sed -i -E "s/[a-zA-Z0-9]+.ubuntu.com/mirrors.aliyun.com/g" /etc/apt/sources.list RUN apt-get clean && apt-get update && apt-get install -y apt-transport-https ca-certificates @@ -23,18 +19,13 @@ RUN apt-get install -y \ librrd-dev \ sudo -# 修改时区 +# timezone modification RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime -# 安装 docker client +# install docker client RUN curl -fsSL "http://resources.koderover.com/docker-cli-v19.03.2.tar.gz" -o docker.tgz &&\ tar -xvzf docker.tgz &&\ mv docker/* /usr/local/bin - -# 替换tar（适配cephfs） +# replace tar（compatible for cephfs） RUN rm /bin/tar && curl -fsSL http://resource.koderover.com/tar -o /bin/tar && chmod +x /bin/tar - -COPY --from=build /reaper /usr/local/bin - -ENTRYPOINT ["reaper"] diff --git a/docker/service/reaper-plugin-bionic.Dockerfile b/docker/base/image/bionic-base-arm64.Dockerfile similarity index 77% rename from docker/service/reaper-plugin-bionic.Dockerfile rename to docker/base/image/bionic-base-arm64.Dockerfile index a5bb8f3e100609c09f4c2eaf48ee180abc288285..3c6f2b796312441920b444a17a43b257228659ec 100644 --- a/docker/service/reaper-plugin-bionic.Dockerfile +++ b/docker/base/image/bionic-base-arm64.Dockerfile @@ -1,8 +1,4 @@ -#golang.Dockerfile - -RUN go build -v -o /reaper ./cmd/reaper/main.go - -#ubuntu-bionic.Dockerfile +FROM arm64v8/ubuntu:bionic RUN sed -i -E "s/[a-zA-Z0-9]+.ubuntu.com/mirrors.aliyun.com/g" /etc/apt/sources.list RUN apt-get clean && apt-get update && apt-get install -y apt-transport-https ca-certificates @@ -23,18 +19,13 @@ RUN apt-get install -y \ librrd-dev \ sudo -# 修改时区 +# timezone modification RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime -# 安装 docker client +# install docker client RUN curl -fsSL "http://resources.koderover.com/docker-cli-v19.03.2.tar.gz" -o docker.tgz &&\ tar -xvzf docker.tgz &&\ mv docker/* /usr/local/bin - -# 替换tar（适配cephfs） +# replace tar（compatible for cephfs） RUN rm /bin/tar && curl -fsSL http://resource.koderover.com/tar -o /bin/tar && chmod +x /bin/tar - -COPY --from=build /reaper /usr/local/bin - -ENTRYPOINT ["reaper"] diff --git a/docker/service/reaper-plugin-xenial.Dockerfile b/docker/base/image/focal-base-amd64.Dockerfile similarity index 58% rename from docker/service/reaper-plugin-xenial.Dockerfile rename to docker/base/image/focal-base-amd64.Dockerfile index 6a65d68242389e7214bd31c89f427fdf1e46c423..0c73eebc59b9e8de7999d57a754ce8c30069121a 100644 --- a/docker/service/reaper-plugin-xenial.Dockerfile +++ b/docker/base/image/focal-base-amd64.Dockerfile @@ -1,15 +1,10 @@ -#golang.Dockerfile - -RUN go build -v -o /reaper ./cmd/reaper/main.go - -#ubuntu-xenial.Dockerfile +FROM ubuntu:focal RUN sed -i -E "s/[a-zA-Z0-9]+.ubuntu.com/mirrors.aliyun.com/g" /etc/apt/sources.list RUN apt-get clean && apt-get update && apt-get install -y apt-transport-https ca-certificates -RUN sed -i s:http:https:g /etc/apt/sources.list RUN DEBIAN_FRONTEND=noninteractive apt install -y tzdata -RUN apt-get clean && apt-get update && apt-get install -y \ - curl \ +RUN apt-get install -y \ + curl \ git \ netcat-openbsd \ wget \ @@ -24,21 +19,13 @@ RUN apt-get clean && apt-get update && apt-get install -y \ librrd-dev \ sudo -# Upgrade Git to latest version -RUN apt-get install -y software-properties-common && add-apt-repository -y ppa:git-core/ppa && apt-get update && apt-get install -y git - -# 修改时区 +# timezone modification RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime -# 安装 docker client +# install docker client RUN curl -fsSL "http://resources.koderover.com/docker-cli-v19.03.2.tar.gz" -o docker.tgz &&\ tar -xvzf docker.tgz &&\ mv docker/* /usr/local/bin - -# 替换tar（适配cephfs） +# replace tar（compatible for cephfs） RUN rm /bin/tar && curl -fsSL http://resource.koderover.com/tar -o /bin/tar && chmod +x /bin/tar - -COPY --from=build /reaper /usr/local/bin - -ENTRYPOINT ["reaper"] diff --git a/docker/base/image/focal-base-arm64.Dockerfile b/docker/base/image/focal-base-arm64.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..360d90faf0ee82c7397d43d6a150cf10e12790c5 --- /dev/null +++ b/docker/base/image/focal-base-arm64.Dockerfile @@ -0,0 +1,31 @@ +FROM arm64v8/ubuntu:focal + +RUN sed -i -E "s/[a-zA-Z0-9]+.ubuntu.com/mirrors.aliyun.com/g" /etc/apt/sources.list +RUN apt-get clean && apt-get update && apt-get install -y apt-transport-https ca-certificates +RUN DEBIAN_FRONTEND=noninteractive apt install -y tzdata +RUN apt-get install -y \ + curl \ + git \ + netcat-openbsd \ + wget \ + build-essential \ + libfontconfig \ + libsasl2-dev \ + libfreetype6-dev \ + libpcre3-dev \ + pkg-config \ + cmake \ + python \ + librrd-dev \ + sudo + +# timezone modification +RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime + +# install docker client +RUN curl -fsSL "http://resources.koderover.com/docker-cli-v19.03.2.tar.gz" -o docker.tgz &&\ + tar -xvzf docker.tgz &&\ + mv docker/* /usr/local/bin + +# replace tar（compatible for cephfs） +RUN rm /bin/tar && curl -fsSL http://resource.koderover.com/tar -o /bin/tar && chmod +x /bin/tar diff --git a/docker/base/image/xenial-base-amd64.Dockerfile b/docker/base/image/xenial-base-amd64.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..27ab04b56cadca931c353862e562385dc7e94aaf --- /dev/null +++ b/docker/base/image/xenial-base-amd64.Dockerfile @@ -0,0 +1,53 @@ +FROM ubuntu:xenial + +RUN sed -i -E "s/[a-zA-Z0-9]+.ubuntu.com/mirrors.aliyun.com/g" /etc/apt/sources.list +RUN apt-get clean && apt-get update && apt-get install -y apt-transport-https ca-certificates +RUN DEBIAN_FRONTEND=noninteractive apt install -y tzdata +RUN apt-get clean && apt-get update && apt-get install -y \ + curl \ + netcat-openbsd \ + wget \ + build-essential \ + libfontconfig \ + libsasl2-dev \ + libfreetype6-dev \ + libpcre3-dev \ + pkg-config \ + cmake \ + python \ + librrd-dev \ + python \ + net-tools \ + dnsutils \ + ca-certificates \ + lsof \ + telnet \ + sudo + +# Upgrade Git to latest version,forcing IPv4 transport with apt-get +RUN echo "deb http://launchpad.proxy.ustclug.org/git-core/ppa/ubuntu xenial main" >> /etc/apt/sources.list + +# Using 'launchpad.proxy.ustclug.org' to reverse proxy for 'launchpad.net'. Detail:https://lug.ustc.edu.cn/wiki/mirrors/help/revproxy/ +RUN echo "deb-src http://launchpad.proxy.ustclug.org/git-core/ppa/ubuntu xenial main" >> /etc/apt/sources.list + +# Import key manually to solve bad network problems +RUN echo '-----BEGIN PGP PUBLIC KEY BLOCK----- \nVersion: \nComment: Hostname: keyserver.ubuntu.com \n \nxo0ESXjaGwEEAMA26F3+mnRW8uRqASMsEa5EsmgvUpLD7EKpC7903OpiMGSvZ2sE \n34g7W6nUQY0R//AZS2iW4ZXfvdhQTQuuPlHM6Q3iUAt+nyXcf9xBlscs8Gm722u4 \njAtFtBS4BMQRhRRfWTHwJIOM6OpGIccjPe8pQfIeoRxkKJxlehzw2mU1ABEBAAHN \nKExhdW5jaHBhZCBQUEEgZm9yIFVidW50dSBHaXQgTWFpbnRhaW5lcnPCtgQTAQIA \nIAUCSXjaGwIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEKFxXYjh3x8k/zMD \n/RKBMjavvFl71YBazSOGl2YfSsZiR/ANsby3+rUaULb8uxzCHXAQnlH5vdtLSPry \naLBvzCU8C3C02qNT8jRacU2752zsCkCi1SLRSOXdI/ATJHza5aTvYV93rTITBhU4 \nsJQeK9RW0CtaDRAxJsn/Dr6J3lL/c9m9cT5fFpxOIsF4 \n=7kFR \n-----END PGP PUBLIC KEY BLOCK----- \n' > key + +RUN apt-key add key + +# Forcing IPv4 transport with apt-get +RUN apt-get -o Acquire::ForceIPv4=true update + +RUN apt-get -o Acquire::ForceIPv4=true install -y git + +# Change timezone to Asia/Shanghai +RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime + +# Install docker client +RUN curl -fsSL "http://resources.koderover.com/docker-cli-v19.03.2.tar.gz" -o docker.tgz &&\ + tar -xvzf docker.tgz &&\ + mv docker/* /usr/local/bin + + +# Replaces the default tar（for cephfs） +RUN rm /bin/tar && curl -fsSL http://resource.koderover.com/tar -o /bin/tar && chmod +x /bin/tar diff --git a/docker/base/image/xenial-base-arm64.Dockerfile b/docker/base/image/xenial-base-arm64.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..00ec977c3b1c830c5d324b42891fc94e695c7197 --- /dev/null +++ b/docker/base/image/xenial-base-arm64.Dockerfile @@ -0,0 +1,53 @@ +FROM arm64v8/ubuntu:xenial + +RUN sed -i -E "s/[a-zA-Z0-9]+.ubuntu.com/mirrors.aliyun.com/g" /etc/apt/sources.list +RUN apt-get clean && apt-get update && apt-get install -y apt-transport-https ca-certificates +RUN DEBIAN_FRONTEND=noninteractive apt install -y tzdata +RUN apt-get clean && apt-get update && apt-get install -y \ + curl \ + netcat-openbsd \ + wget \ + build-essential \ + libfontconfig \ + libsasl2-dev \ + libfreetype6-dev \ + libpcre3-dev \ + pkg-config \ + cmake \ + python \ + librrd-dev \ + python \ + net-tools \ + dnsutils \ + ca-certificates \ + lsof \ + telnet \ + sudo + +# Upgrade Git to latest version,forcing IPv4 transport with apt-get +RUN echo "deb http://launchpad.proxy.ustclug.org/git-core/ppa/ubuntu xenial main" >> /etc/apt/sources.list + +# Using 'launchpad.proxy.ustclug.org' to reverse proxy for 'launchpad.net'. Detail:https://lug.ustc.edu.cn/wiki/mirrors/help/revproxy/ +RUN echo "deb-src http://launchpad.proxy.ustclug.org/git-core/ppa/ubuntu xenial main" >> /etc/apt/sources.list + +# Import key manually to solve bad network problems +RUN echo '-----BEGIN PGP PUBLIC KEY BLOCK----- \nVersion: \nComment: Hostname: keyserver.ubuntu.com \n \nxo0ESXjaGwEEAMA26F3+mnRW8uRqASMsEa5EsmgvUpLD7EKpC7903OpiMGSvZ2sE \n34g7W6nUQY0R//AZS2iW4ZXfvdhQTQuuPlHM6Q3iUAt+nyXcf9xBlscs8Gm722u4 \njAtFtBS4BMQRhRRfWTHwJIOM6OpGIccjPe8pQfIeoRxkKJxlehzw2mU1ABEBAAHN \nKExhdW5jaHBhZCBQUEEgZm9yIFVidW50dSBHaXQgTWFpbnRhaW5lcnPCtgQTAQIA \nIAUCSXjaGwIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEKFxXYjh3x8k/zMD \n/RKBMjavvFl71YBazSOGl2YfSsZiR/ANsby3+rUaULb8uxzCHXAQnlH5vdtLSPry \naLBvzCU8C3C02qNT8jRacU2752zsCkCi1SLRSOXdI/ATJHza5aTvYV93rTITBhU4 \nsJQeK9RW0CtaDRAxJsn/Dr6J3lL/c9m9cT5fFpxOIsF4 \n=7kFR \n-----END PGP PUBLIC KEY BLOCK----- \n' > key + +RUN apt-key add key + +# Forcing IPv4 transport with apt-get +RUN apt-get -o Acquire::ForceIPv4=true update + +RUN apt-get -o Acquire::ForceIPv4=true install -y git + +# Change timezone to Asia/Shanghai +RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime + +# Install docker client +RUN curl -fsSL "http://resources.koderover.com/docker-cli-v19.03.2.tar.gz" -o docker.tgz &&\ + tar -xvzf docker.tgz &&\ + mv docker/* /usr/local/bin + + +# Replaces the default tar（for cephfs） +RUN rm /bin/tar && curl -fsSL http://resource.koderover.com/tar -o /bin/tar && chmod +x /bin/tar diff --git a/docker/service/aslan.Dockerfile b/docker/service/aslan.Dockerfile index 434810faaca611b36e29a8d1473efefec82d6c90..f8d84fdb41bf504931ab87de217f22c44df6cf30 100644 --- a/docker/service/aslan.Dockerfile +++ b/docker/service/aslan.Dockerfile @@ -1,11 +1,15 @@ -#golang.Dockerfile +#alpine-git.Dockerfile -RUN go build -v -o /aslan ./cmd/aslan/main.go +RUN apk --no-cache add curl -#alpine-git.Dockerfile +# install ali-acr plugin +RUN curl -fsSL "https://resources.koderover.com/helm-acr_0.8.2_linux_amd64.tar.gz" -o helm-acr.tar.gz &&\ + mkdir -p /app/.helm/helmplugin/helm-acr &&\ + tar -xvzf helm-acr.tar.gz -C /app/.helm/helmplugin/helm-acr &&\ + rm -rf helm-acr* WORKDIR /app -COPY --from=build /aslan . +ADD docker/dist/aslan . ENTRYPOINT ["/app/aslan"] diff --git a/docker/service/config.Dockerfile b/docker/service/config.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..bdb7a8e48294e2bfac457cf08fb8bfc778a20f4d --- /dev/null +++ b/docker/service/config.Dockerfile @@ -0,0 +1,7 @@ +#alpine.Dockerfile + +WORKDIR /app + +ADD docker/dist/config . + +ENTRYPOINT ["/app/config"] diff --git a/docker/service/cron.Dockerfile b/docker/service/cron.Dockerfile index b0c60a1bd368fe8324392a74ee195c696c37e497..162acad47757d9cdcf48de91c25f30a022d638d4 100644 --- a/docker/service/cron.Dockerfile +++ b/docker/service/cron.Dockerfile @@ -1,11 +1,7 @@ -#golang.Dockerfile - -RUN go build -v -o /cron ./cmd/cron/main.go - #alpine-git.Dockerfile WORKDIR /app -COPY --from=build /cron . +ADD docker/dist/cron . ENTRYPOINT ["/app/cron"] diff --git a/docker/service/hub-agent.Dockerfile b/docker/service/hub-agent.Dockerfile index 4113806170843c703da1ed1d19f0570db8736626..f69cb6fd8bd421c098e8b68a008bfad66d8b7424 100644 --- a/docker/service/hub-agent.Dockerfile +++ b/docker/service/hub-agent.Dockerfile @@ -1,11 +1,7 @@ -#golang.Dockerfile - -RUN go build -v -o /hubagent ./cmd/hubagent/main.go - #alpine-git.Dockerfile WORKDIR /app -COPY --from=build /hubagent . +ADD docker/dist/hub-agent . -ENTRYPOINT ["/app/hubagent"] +ENTRYPOINT ["/app/hub-agent"] diff --git a/docker/service/hub-server.Dockerfile b/docker/service/hub-server.Dockerfile index 7bb4ae3e345f0f8e33314a4dde56c75492e506d4..ba31683702b2569384c33bc903c7d0e5fa8565c9 100644 --- a/docker/service/hub-server.Dockerfile +++ b/docker/service/hub-server.Dockerfile @@ -1,11 +1,7 @@ -#golang.Dockerfile - -RUN go build -v -o /hubserver ./cmd/hubserver/main.go - #alpine.Dockerfile WORKDIR /app -COPY --from=build /hubserver . +ADD docker/dist/hub-server . -ENTRYPOINT ["/app/hubserver"] +ENTRYPOINT ["/app/hub-server"] diff --git a/docker/service/init.Dockerfile b/docker/service/init.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..be650b5a6adf1e03d8dfb5efc9a3efefcd7e91c8 --- /dev/null +++ b/docker/service/init.Dockerfile @@ -0,0 +1,7 @@ +#alpine.Dockerfile + +WORKDIR /app + +ADD docker/dist/init /app/init + +ENTRYPOINT ["/app/init"] diff --git a/docker/service/jenkins-plugin.Dockerfile b/docker/service/jenkins-plugin.Dockerfile index 4fd1ec66441677eff685995d9011f9baf2777acd..0b6bd98881deb66f4317ba40fd1b3df64bf27441 100644 --- a/docker/service/jenkins-plugin.Dockerfile +++ b/docker/service/jenkins-plugin.Dockerfile @@ -1,11 +1,7 @@ -#golang.Dockerfile - -RUN go build -v -o /jenkins-plugin ./cmd/jenkinsplugin/main.go - #alpine.Dockerfile WORKDIR /app -COPY --from=build /jenkins-plugin . +ADD docker/dist/jenkins-plugin . ENTRYPOINT ["/app/jenkins-plugin"] diff --git a/docker/service/packager-plugin.Dockerfile b/docker/service/packager-plugin.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..d5f329611f7a5203a55aa0e1de1b218fb4fe1590 --- /dev/null +++ b/docker/service/packager-plugin.Dockerfile @@ -0,0 +1,7 @@ +#alpine.Dockerfile + +WORKDIR /app + +ADD docker/dist/packager-plugin . + +ENTRYPOINT ["/app/packager-plugin"] diff --git a/docker/service/picket.Dockerfile b/docker/service/picket.Dockerfile index 09a92d7feddbc8b2d0c99ec98800377bb9f4d96c..242375f5987eecc43e8e7d13f85fb8ab118c667b 100644 --- a/docker/service/picket.Dockerfile +++ b/docker/service/picket.Dockerfile @@ -1,11 +1,7 @@ -#golang.Dockerfile - -RUN go build -v -o /picket ./cmd/picket/main.go - #alpine.Dockerfile WORKDIR /app -COPY --from=build /picket . +ADD docker/dist/picket . ENTRYPOINT ["/app/picket"] diff --git a/docker/service/podexec.Dockerfile b/docker/service/podexec.Dockerfile index 657bfae66a6c84a0e18bc3f067ac1c4c9186bf0a..1ef16630d9b592726187dcb1a610e2dee95d42dd 100644 --- a/docker/service/podexec.Dockerfile +++ b/docker/service/podexec.Dockerfile @@ -1,10 +1,7 @@ -#golang.Dockerfile - -RUN go build -v -o /podexec ./cmd/podexec/... - #alpine-git.Dockerfile WORKDIR /app -COPY --from=build /podexec /app/podexec + +ADD docker/dist/podexec . ENTRYPOINT ["/app/podexec"] diff --git a/docker/service/policy.Dockerfile b/docker/service/policy.Dockerfile index bb8314397ab61af41a6a5f375a1410de14fcfb62..bd3cc58c05012614bcb81e7fb471c7476bd90081 100644 --- a/docker/service/policy.Dockerfile +++ b/docker/service/policy.Dockerfile @@ -1,11 +1,7 @@ -#golang.Dockerfile - -RUN go build -v -o /policy ./cmd/policy/main.go - #alpine.Dockerfile WORKDIR /app -COPY --from=build /policy . +ADD docker/dist/policy . ENTRYPOINT ["/app/policy"] diff --git a/docker/service/predator-plugin.Dockerfile b/docker/service/predator-plugin.Dockerfile index edc2f1c6edc23b30f0e2b347439c0dd4318a5ef5..30f7b385b0e419137f40a32ec4cf3eece4ae08be 100644 --- a/docker/service/predator-plugin.Dockerfile +++ b/docker/service/predator-plugin.Dockerfile @@ -1,10 +1,6 @@ -#golang.Dockerfile - -RUN go build -v -o /predator-plugin ./cmd/predator/main.go - #ubuntu-xenial.Dockerfile -# 安装 docker client +# install docker client RUN curl -fsSL "http://resources.koderover.com/docker-cli-v19.03.2.tar.gz" -o docker.tgz &&\ tar -xvzf docker.tgz &&\ mv docker/* /usr/local/bin &&\ @@ -12,6 +8,6 @@ RUN curl -fsSL "http://resources.koderover.com/docker-cli-v19.03.2.tar.gz" -o do WORKDIR /app -COPY --from=build /predator-plugin . +ADD docker/dist/predator-plugin . ENTRYPOINT ["/app/predator-plugin"] diff --git a/docker/service/resource-server.Dockerfile b/docker/service/resource-server.Dockerfile index 2eb253c29395e23a8fc6bf27ba5c0c96431bc8a6..89b81306cf9dc6748cc208560ee74f84bef09990 100644 --- a/docker/service/resource-server.Dockerfile +++ b/docker/service/resource-server.Dockerfile @@ -1,12 +1,9 @@ -#golang.Dockerfile - -RUN go build -o /reaper ./cmd/reaper/main.go - #nginx.Dockerfile WORKDIR /app ADD resource-server-nginx.conf /etc/nginx/conf.d/default.conf -COPY --from=build /reaper . +ADD docker/dist/reaper . +COPY docker/dist/jobexecutor . EXPOSE 80 diff --git a/docker/service/syncthing.Dockerfile b/docker/service/syncthing.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..263aafcae3a825b133bcc7d9d50bed76546016cc --- /dev/null +++ b/docker/service/syncthing.Dockerfile @@ -0,0 +1,3 @@ +FROM syncthing/syncthing:1.20.1 + +WORKDIR /home/syncthing diff --git a/docker/service/ua.Dockerfile b/docker/service/ua.Dockerfile index f3b7a43a17386ef53e3f083856bd5ebd665e7779..6b0323f570029934f03897500495d6c3784de0a7 100644 --- a/docker/service/ua.Dockerfile +++ b/docker/service/ua.Dockerfile @@ -1,10 +1,7 @@ -#golang.Dockerfile - -RUN go build -o /ua ./cmd/upgradeassistant/main.go - #alpine.Dockerfile WORKDIR /app -COPY --from=build /ua /app/ua + +ADD docker/dist/ua . ENTRYPOINT ["/app/ua"] \ No newline at end of file diff --git a/docker/service/user.Dockerfile b/docker/service/user.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..044c398b0bec76208253719693a3bd4148337644 --- /dev/null +++ b/docker/service/user.Dockerfile @@ -0,0 +1,7 @@ +#alpine.Dockerfile + +WORKDIR /app + +ADD docker/dist/user . + +ENTRYPOINT ["/app/user"] diff --git a/docker/service/warpdrive.Dockerfile b/docker/service/warpdrive.Dockerfile index f7d0a44a9cbe2dafc0c515d8bfe45a2feb659108..767b01c1aa71070fa2b5e205d05a29d78ee63011 100644 --- a/docker/service/warpdrive.Dockerfile +++ b/docker/service/warpdrive.Dockerfile @@ -1,11 +1,7 @@ -#golang.Dockerfile - -RUN go build -v -o /warpdrive ./cmd/warpdrive/main.go - #alpine.Dockerfile WORKDIR /app -COPY --from=build /warpdrive . +COPY docker/dist/warpdrive . ENTRYPOINT ["/app/warpdrive"] diff --git a/docker/service/zadig-debug.Dockerfile b/docker/service/zadig-debug.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..192a4e6c2d688baedc22f65102d9a1d680f96bb4 --- /dev/null +++ b/docker/service/zadig-debug.Dockerfile @@ -0,0 +1,4 @@ +FROM alpine:3.16.0 + +RUN apk update && apk upgrade +RUN apk add bash curl wget iputils busybox-extras bind-tools tcpdump net-tools procps sysstat diff --git a/docker/ua.Dockerfile.template b/docker/ua.Dockerfile.template deleted file mode 100644 index 184841993e444f396269faf6cad315458c20cc6c..0000000000000000000000000000000000000000 --- a/docker/ua.Dockerfile.template +++ /dev/null @@ -1,10 +0,0 @@ -#golang-deps.Dockerfile.inc - -RUN go build -o /ua ./cmd/upgradeassistant/main.go - -#alpine-base.Dockerfile.inc - -WORKDIR /app -COPY --from=build /ua /app/ua - -ENTRYPOINT ["/app/ua"] diff --git a/examples/configmap-demo.yaml b/examples/configmap-demo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb628122862b77f584f4464d98845e9f259c258d --- /dev/null +++ b/examples/configmap-demo.yaml @@ -0,0 +1,15 @@ +# test +apiVersion: v1 +kind: ConfigMap +metadata: + name: game-demo +data: + player_initial_lives: "1" + ui_properties_file_name: "user-interface.properties" + game.properties: | + enemy.types=aliens,monsters + player.maximum-lives=1 + user-interface.properties: | + color.good=purple1 + color.bad=yellow1 + allow.textmode=true1 \ No newline at end of file diff --git a/examples/jMeter-demo/demo.jmx b/examples/jMeter-demo/demo.jmx new file mode 100644 index 0000000000000000000000000000000000000000..b65b71732e7c58a6e821be140c05324241ca1f4b --- /dev/null +++ b/examples/jMeter-demo/demo.jmx @@ -0,0 +1,51 @@ + + + + + + false + true + false + + + + + + + + continue + + false + 1 + + 10 + 1 + false + + + true + + + + + + + www.koderover.com + + https + + / + GET + true + false + true + false + + + + + + + + + diff --git a/examples/microservice-demo/backend/backend.go b/examples/microservice-demo/backend/backend.go index eccde3bbc9ce3d7500bd3b0c0e4dc72ec2806eab..978dfb0509a71555155f0c68c3f154cdb97b0beb 100644 --- a/examples/microservice-demo/backend/backend.go +++ b/examples/microservice-demo/backend/backend.go @@ -18,6 +18,7 @@ package main import ( "fmt" + "log" "net/http" ) @@ -44,6 +45,8 @@ func buildStamp(w http.ResponseWriter, req *http.Request) { func main() { + log.Println("Hello, welcome to the microservice world.") + http.HandleFunc("/", hello) http.HandleFunc("/api/buildstamp", buildStamp) http.HandleFunc("/headers", headers) diff --git a/examples/microservice-demo/backend/go.mod b/examples/microservice-demo/backend/go.mod new file mode 100644 index 0000000000000000000000000000000000000000..2090ee85822f11e43fbdb97598567bc06ecea35f --- /dev/null +++ b/examples/microservice-demo/backend/go.mod @@ -0,0 +1,3 @@ +module microservice-demo/backend + +go 1.17 diff --git a/examples/microservice-demo/chart/templates/backend.yaml b/examples/microservice-demo/chart/templates/backend.yaml index 0f40aa6ac681a5a451068f947fbffd590b9bc43a..fd2b2e98ed0971fa4d2ea6ea0537c2314448db97 100644 --- a/examples/microservice-demo/chart/templates/backend.yaml +++ b/examples/microservice-demo/chart/templates/backend.yaml @@ -26,6 +26,10 @@ spec: ports: - protocol: TCP containerPort: 20219 + resources: + limits: + cpu: {{ .Values.resources.limits.cpu }} + memory: {{ .Values.resources.limits.memory }} imagePullSecrets: - name: registry-secret --- diff --git a/examples/microservice-demo/chart/templates/frontend.yaml b/examples/microservice-demo/chart/templates/frontend.yaml index 694148d725dc3378465bb33b02eea553f7dc43bb..fe38eae05053eed50705fa4d9aaab012e25fdec9 100644 --- a/examples/microservice-demo/chart/templates/frontend.yaml +++ b/examples/microservice-demo/chart/templates/frontend.yaml @@ -31,11 +31,8 @@ spec: containerPort: 80 resources: limits: - cpu: 1 - memory: 512Mi - requests: - cpu: 100m - memory: 100M + cpu: {{ .Values.resources.limits.cpu }} + memory: {{ .Values.resources.limits.memory }} imagePullSecrets: - name: registry-secret --- @@ -56,7 +53,7 @@ spec: port: 80 targetPort: 80 --- -apiVersion: extensions/v1beta1 +apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: frontend @@ -73,9 +70,9 @@ spec: http: paths: - path: / + pathType: Prefix backend: - serviceName: frontend - servicePort: 80 - - - + service: + name: frontend + port: + number: 80 diff --git a/examples/microservice-demo/chart/values.yaml b/examples/microservice-demo/chart/values.yaml index 9b7c7d2610f6638bacf5c646e94491767f97ec0c..b3b78d71d1b2839cb841a536b6747c7e0be2fe33 100644 --- a/examples/microservice-demo/chart/values.yaml +++ b/examples/microservice-demo/chart/values.yaml @@ -6,16 +6,21 @@ ingressClass: kr-nginx Backend: replicaCount: 1 image: - repository: ccr.ccs.tencentyun.com/koderover-public/backend + repository: koderover.tencentcloudcr.com/koderover-demo/backend tag: "latest" Frontend: replicaCount: 1 image: - repository: ccr.ccs.tencentyun.com/koderover-public/frontend + repository: koderover.tencentcloudcr.com/koderover-demo/frontend tag: "latest" domain: demo.domain.com port: 20219 +resources: + limits: + cpu: 50m + memory: 50Mi + registrySecret: "" diff --git a/examples/microservice-demo/frontend/Dockerfile b/examples/microservice-demo/frontend/Dockerfile index 1d287cf6549a5916228db7a92d41671c82d2f8ca..0c0dcfd6cb4b545cf7f96e8f6ed3d38fb0757c54 100644 --- a/examples/microservice-demo/frontend/Dockerfile +++ b/examples/microservice-demo/frontend/Dockerfile @@ -1,4 +1,4 @@ -FROM ccr.ccs.tencentyun.com/trial-dist/frontend-base:0.1 As builder +FROM koderover.tencentcloudcr.com/koderover-demo/frontend-base:0.1 As builder WORKDIR /frontend diff --git a/examples/microservice-demo/k8s-yaml/backend/deployment.yaml b/examples/microservice-demo/k8s-yaml/backend/deployment.yaml index b5e0fc13d8ef7f4be3536114835f7eba8a2a784f..a1ddb6b0ae6b948071eb5ecc3ce7c73f500bffbf 100644 --- a/examples/microservice-demo/k8s-yaml/backend/deployment.yaml +++ b/examples/microservice-demo/k8s-yaml/backend/deployment.yaml @@ -19,11 +19,16 @@ spec: spec: containers: - name: backend - image: ccr.ccs.tencentyun.com/koderover-public/backend:latest - imagePullPolicy: Always + image: koderover.tencentcloudcr.com/koderover-demo/backend:latest + imagePullPolicy: Always command: - /workspace/backend ports: - protocol: TCP containerPort: 20219 - \ No newline at end of file + resources: + limits: + memory: 50Mi + cpu: 50m + imagePullSecrets: + - name: default-registry-secret diff --git a/examples/microservice-demo/k8s-yaml/frontend/deployment.yaml b/examples/microservice-demo/k8s-yaml/frontend/deployment.yaml index ee894c2ce1986d30e06cfe78c675076283ea9600..1ca9547ad52e63442af39f4a30f20e772ed404cc 100644 --- a/examples/microservice-demo/k8s-yaml/frontend/deployment.yaml +++ b/examples/microservice-demo/k8s-yaml/frontend/deployment.yaml @@ -24,18 +24,14 @@ spec: spec: containers: - name: frontend - image: ccr.ccs.tencentyun.com/koderover-public/frontend:latest + image: koderover.tencentcloudcr.com/koderover-demo/frontend:latest imagePullPolicy: Always ports: - protocol: TCP containerPort: 80 resources: limits: - cpu: 1 - memory: 512Mi - requests: cpu: 100m memory: 100M - - - + imagePullSecrets: + - name: default-registry-secret diff --git a/examples/microservice-demo/k8s-yaml/frontend/ingress.yaml b/examples/microservice-demo/k8s-yaml/frontend/ingress.yaml index bf98292d85f78989c2c8c153445cf07c72a3697d..8596d5f6a44122f07555a25fe97fcc19183f2601 100644 --- a/examples/microservice-demo/k8s-yaml/frontend/ingress.yaml +++ b/examples/microservice-demo/k8s-yaml/frontend/ingress.yaml @@ -1,4 +1,4 @@ -apiVersion: extensions/v1beta1 +apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: frontend @@ -13,6 +13,9 @@ spec: http: paths: - path: / + pathType: Prefix backend: - serviceName: frontend - servicePort: 80 \ No newline at end of file + service: + name: frontend + port: + number: 80 \ No newline at end of file diff --git a/examples/microservice-demo/vm/ansible/ansible.Dockerfile b/examples/microservice-demo/vm/ansible/ansible.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..b3cd73c6c4789494f5575544dbdc8b68a11a305a --- /dev/null +++ b/examples/microservice-demo/vm/ansible/ansible.Dockerfile @@ -0,0 +1,22 @@ +FROM ubuntu:focal + +RUN apt update +RUN apt install -y software-properties-common +RUN add-apt-repository --yes --update ppa:ansible/ansible +RUN apt install -y ansible + +# 以下是 Zadig 所需要的依赖 + +# 安装工具 curl git tzdata +RUN sed -i -E "s/[a-zA-Z0-9]+.ubuntu.com/mirrors.aliyun.com/g" /etc/apt/sources.list +RUN apt-get clean && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y curl git tzdata + +# 修改时区 +RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime + +# 安装 docker client +RUN curl -fsSL "http://resources.koderover.com/docker-cli-v19.03.2.tar.gz" -o docker.tgz &&\ + tar -xvzf docker.tgz &&\ + mv docker/* /usr/local/bin + +RUN apt install -y build-essential net-tools diff --git a/examples/microservice-demo/vm/ansible/hosts-dev b/examples/microservice-demo/vm/ansible/hosts-dev new file mode 100644 index 0000000000000000000000000000000000000000..e1595da8d42918a017d632b2cd5c94efc641a89b --- /dev/null +++ b/examples/microservice-demo/vm/ansible/hosts-dev @@ -0,0 +1,5 @@ +[servers] +220.16.0.1 +220.16.0.2 +220.16.0.3 +220.16.0.4 diff --git a/examples/microservice-demo/vm/ansible/hosts-qa b/examples/microservice-demo/vm/ansible/hosts-qa new file mode 100644 index 0000000000000000000000000000000000000000..703850adbacac3a9cb7e47e2236ff9dd7a50a403 --- /dev/null +++ b/examples/microservice-demo/vm/ansible/hosts-qa @@ -0,0 +1,5 @@ +[servers] +220.16.0.11 +220.16.0.12 +220.16.0.13 +220.16.0.14 diff --git a/examples/microservice-demo/vm/ansible/main.yaml b/examples/microservice-demo/vm/ansible/main.yaml new file mode 100644 index 0000000000000000000000000000000000000000..af24bd40588a8cdf23b7cf002dfb1c71b9a70343 --- /dev/null +++ b/examples/microservice-demo/vm/ansible/main.yaml @@ -0,0 +1,14 @@ +--- +- hosts: servers + remote_user: ubuntu + tasks: + - name: "copy file to remote servers" + copy: + src: "{{ PKG_FILE }}" + dest: "/home/ubuntu/microservice-demo/{{ PKG_FILE }}" + - name: "unzip file" + shell: + chdir: /home/ubuntu/microservice-demo + cmd: tar -xvf /home/ubuntu/microservice-demo/{{ PKG_FILE }} + - name: "deploy backend service" + command: /home/ubuntu/microservice-demo/restart.sh /home/ubuntu/microservice-demo/{{ PKG_FILE }} diff --git a/examples/multi-service-demo/Dockerfile b/examples/multi-service-demo/Dockerfile index bb035846e97b964c41f6fff08c7d55c1e40de1cb..bdab9f3a59fecac4ef81570bece486731687b2d6 100644 --- a/examples/multi-service-demo/Dockerfile +++ b/examples/multi-service-demo/Dockerfile @@ -4,6 +4,4 @@ ARG service WORKDIR /workspace -COPY ${service}/${service} /workspace/${service} - -CMD [ "./${service}" ] \ No newline at end of file +COPY src/${service}/${service} /workspace/${service} \ No newline at end of file diff --git a/examples/multi-service-demo/Makefile b/examples/multi-service-demo/Makefile index e2f9cd6ac31d47a220b58f15c8d3ebc0b22e4b1e..ceda2f2915b8a962279ebf126200b09a44d1ea8e 100644 --- a/examples/multi-service-demo/Makefile +++ b/examples/multi-service-demo/Makefile @@ -1,6 +1,10 @@ - build-service1: - cd service1 && go clean && GOOS=linux GOARCH=amd64 go build -o service1 service1.go + cd src/service1 && go clean && GOOS=linux GOARCH=amd64 go build -o service1 service1.go + build-service2: - cd service2 && go clean && GOOS=linux GOARCH=amd64 go build -o service2 service2.go \ No newline at end of file + cd src/service2 && go clean && GOOS=linux GOARCH=amd64 go build -o service2 service2.go + + +build-service3: + cd src/service3 && go clean && GOOS=linux GOARCH=amd64 go build -o service3 service3.go diff --git a/examples/multi-service-demo/README.md b/examples/multi-service-demo/README.md new file mode 100644 index 0000000000000000000000000000000000000000..90968f16ac26ecb160699d34153264c4b9eb751c --- /dev/null +++ b/examples/multi-service-demo/README.md @@ -0,0 +1,16 @@ +`multi-service-demo` 目录结构说明： + +``` bash +├── Dockerfile # Dockerfile，也可以作为 Dockerfile 模板，构建镜像时需要通过 --build-arg service=<值> 指定 service 参数的值 +├── Makefile # 构建工程文件，构建时使用 make build-service* 即可 +├── base-chart # Helm Chart 模板，可用于使用模板批量创建服务 +├── full-charts # 多个 Helm Chart，service1、service2、service3 分别有完整独立的 Helm Chart 配置 +├── general-chart # Helm Chart 模板，不可用于使用模板批量创建服务 +├── k8s-yaml # K8s YAML 配置文件，用于 K8s YAML 项目 +│   ├── service1 # service1 完整的配置 +│   ├── service2 # service2 完整的配置 +│   ├── service3 # service3 完整的配置 +│   └── template.yaml # service1/service2/service3 的 K8s YAML 服务模板 +├── src # 服务源代码 +└── values # 使用 base-chart 模板批量创建服务时，多个服务的 values 文件 +``` diff --git a/examples/multi-service-demo/base-chart/Chart.yaml b/examples/multi-service-demo/base-chart/Chart.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08e787e5acc8e1c120b74f1e50f7d0ad4a9e1675 --- /dev/null +++ b/examples/multi-service-demo/base-chart/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: general-chart +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/examples/multi-service-demo/charts/service1/templates/_helpers.tpl b/examples/multi-service-demo/base-chart/templates/_helpers.tpl similarity index 100% rename from examples/multi-service-demo/charts/service1/templates/_helpers.tpl rename to examples/multi-service-demo/base-chart/templates/_helpers.tpl diff --git a/examples/multi-service-demo/charts/service2/templates/deployment.yaml b/examples/multi-service-demo/base-chart/templates/deployment.yaml similarity index 86% rename from examples/multi-service-demo/charts/service2/templates/deployment.yaml rename to examples/multi-service-demo/base-chart/templates/deployment.yaml index 2ecbd1916e6bd4e4655d7ad634453c443a1b8e84..e441d6b85ac4b9f2a9d9c5e0f0ad05d3fa89a918 100644 --- a/examples/multi-service-demo/charts/service2/templates/deployment.yaml +++ b/examples/multi-service-demo/base-chart/templates/deployment.yaml @@ -29,10 +29,9 @@ spec: resources: limits: cpu: {{ .Values.resources.limits.cpu }} - memory: {{ .Values.resources.limits.memory }} + memory: {{ .Values.resources.limits.mem }} requests: cpu: {{ .Values.resources.requests.cpu }} - memory: {{ .Values.resources.requests.memory }} + memory: {{ .Values.resources.requests.mem }} imagePullSecrets: - - name: {{ .Values.imagePullSecrets }} - + - name: {{ .Values.imagePullSecretsName }} \ No newline at end of file diff --git a/examples/multi-service-demo/charts/service1/templates/service.yaml b/examples/multi-service-demo/base-chart/templates/service.yaml similarity index 100% rename from examples/multi-service-demo/charts/service1/templates/service.yaml rename to examples/multi-service-demo/base-chart/templates/service.yaml diff --git a/examples/multi-service-demo/base-chart/values.yaml b/examples/multi-service-demo/base-chart/values.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0642e90dcf7167f2b93863571ef247777fcd19d0 --- /dev/null +++ b/examples/multi-service-demo/base-chart/values.yaml @@ -0,0 +1,18 @@ +fullnameOverride: service +replicaCount: 1 +port: 20221 + +imagePullSecretsName: "default-registry-secret" + +image: + repository: "koderover.tencentcloudcr.com/koderover-demo/service" + tag: "latest" + +resources: + requests: + cpu: 10m + mem: 10Mi + + limits: + cpu: 20m + mem: 20Mi diff --git a/examples/multi-service-demo/charts/service1/.helmignore b/examples/multi-service-demo/charts/service1/.helmignore deleted file mode 100644 index 0e8a0eb36f4ca2c939201c0d54b5d82a1ea34778..0000000000000000000000000000000000000000 --- a/examples/multi-service-demo/charts/service1/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/examples/multi-service-demo/charts/service1/templates/secret.yaml b/examples/multi-service-demo/charts/service1/templates/secret.yaml deleted file mode 100644 index cd16a681e4bdacd0d5c74979be07251b540161d2..0000000000000000000000000000000000000000 --- a/examples/multi-service-demo/charts/service1/templates/secret.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -data: - .dockercfg: {{ .Values.registrySecret }} -kind: Secret -metadata: - name: {{ .Values.imagePullSecrets }} -type: kubernetes.io/dockercfg \ No newline at end of file diff --git a/examples/multi-service-demo/charts/service2/.helmignore b/examples/multi-service-demo/charts/service2/.helmignore deleted file mode 100644 index 0e8a0eb36f4ca2c939201c0d54b5d82a1ea34778..0000000000000000000000000000000000000000 --- a/examples/multi-service-demo/charts/service2/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/examples/multi-service-demo/charts/service2/templates/secret.yaml b/examples/multi-service-demo/charts/service2/templates/secret.yaml deleted file mode 100644 index cd16a681e4bdacd0d5c74979be07251b540161d2..0000000000000000000000000000000000000000 --- a/examples/multi-service-demo/charts/service2/templates/secret.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -data: - .dockercfg: {{ .Values.registrySecret }} -kind: Secret -metadata: - name: {{ .Values.imagePullSecrets }} -type: kubernetes.io/dockercfg \ No newline at end of file diff --git a/examples/multi-service-demo/charts/service1/Chart.yaml b/examples/multi-service-demo/full-charts/service1/Chart.yaml similarity index 100% rename from examples/multi-service-demo/charts/service1/Chart.yaml rename to examples/multi-service-demo/full-charts/service1/Chart.yaml diff --git a/examples/multi-service-demo/charts/service2/templates/_helpers.tpl b/examples/multi-service-demo/full-charts/service1/templates/_helpers.tpl similarity index 100% rename from examples/multi-service-demo/charts/service2/templates/_helpers.tpl rename to examples/multi-service-demo/full-charts/service1/templates/_helpers.tpl diff --git a/examples/multi-service-demo/charts/service1/templates/deployment.yaml b/examples/multi-service-demo/full-charts/service1/templates/deployment.yaml similarity index 86% rename from examples/multi-service-demo/charts/service1/templates/deployment.yaml rename to examples/multi-service-demo/full-charts/service1/templates/deployment.yaml index 2ecbd1916e6bd4e4655d7ad634453c443a1b8e84..e441d6b85ac4b9f2a9d9c5e0f0ad05d3fa89a918 100644 --- a/examples/multi-service-demo/charts/service1/templates/deployment.yaml +++ b/examples/multi-service-demo/full-charts/service1/templates/deployment.yaml @@ -29,10 +29,9 @@ spec: resources: limits: cpu: {{ .Values.resources.limits.cpu }} - memory: {{ .Values.resources.limits.memory }} + memory: {{ .Values.resources.limits.mem }} requests: cpu: {{ .Values.resources.requests.cpu }} - memory: {{ .Values.resources.requests.memory }} + memory: {{ .Values.resources.requests.mem }} imagePullSecrets: - - name: {{ .Values.imagePullSecrets }} - + - name: {{ .Values.imagePullSecretsName }} \ No newline at end of file diff --git a/examples/multi-service-demo/charts/service2/templates/service.yaml b/examples/multi-service-demo/full-charts/service1/templates/service.yaml similarity index 100% rename from examples/multi-service-demo/charts/service2/templates/service.yaml rename to examples/multi-service-demo/full-charts/service1/templates/service.yaml diff --git a/examples/multi-service-demo/full-charts/service1/values.yaml b/examples/multi-service-demo/full-charts/service1/values.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a1b782e2996f68bace458a8f46cd98614324e65 --- /dev/null +++ b/examples/multi-service-demo/full-charts/service1/values.yaml @@ -0,0 +1,18 @@ +fullnameOverride: service1 +replicaCount: 1 +port: 20221 + +imagePullSecretsName: "default-registry-secret" + +image: + repository: "koderover.tencentcloudcr.com/koderover-demo/service1" + tag: "latest" + +resources: + requests: + cpu: 10m + mem: 10Mi + + limits: + cpu: 20m + mem: 20Mi diff --git a/examples/multi-service-demo/charts/service2/Chart.yaml b/examples/multi-service-demo/full-charts/service2/Chart.yaml similarity index 100% rename from examples/multi-service-demo/charts/service2/Chart.yaml rename to examples/multi-service-demo/full-charts/service2/Chart.yaml diff --git a/examples/multi-service-demo/full-charts/service2/templates/_helpers.tpl b/examples/multi-service-demo/full-charts/service2/templates/_helpers.tpl new file mode 100644 index 0000000000000000000000000000000000000000..2668feb5bc8a25ef24b51c7fd3c4d9a7d1dcadf5 --- /dev/null +++ b/examples/multi-service-demo/full-charts/service2/templates/_helpers.tpl @@ -0,0 +1,63 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "chart.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "chart.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "chart.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "chart.labels" -}} +helm.sh/chart: {{ include "chart.chart" . }} +{{ include "chart.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "chart.selectorLabels" -}} +app.kubernetes.io/name: {{ include "chart.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "chart.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "chart.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/examples/multi-service-demo/full-charts/service2/templates/deployment.yaml b/examples/multi-service-demo/full-charts/service2/templates/deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e441d6b85ac4b9f2a9d9c5e0f0ad05d3fa89a918 --- /dev/null +++ b/examples/multi-service-demo/full-charts/service2/templates/deployment.yaml @@ -0,0 +1,37 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.fullnameOverride }} + labels: + app.kubernetes.io/name: demo + app.kubernetes.io/instance: {{ .Values.fullnameOverride }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: demo + app.kubernetes.io/instance: {{ .Values.fullnameOverride }} + replicas: {{ .Values.replicaCount }} + template: + metadata: + labels: + app.kubernetes.io/name: demo + app.kubernetes.io/instance: {{ .Values.fullnameOverride }} + spec: + containers: + - name: {{ .Values.fullnameOverride }} + image: {{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: Always + command: + - /workspace/{{ .Values.fullnameOverride }} + ports: + - protocol: TCP + containerPort: {{ .Values.port }} + resources: + limits: + cpu: {{ .Values.resources.limits.cpu }} + memory: {{ .Values.resources.limits.mem }} + requests: + cpu: {{ .Values.resources.requests.cpu }} + memory: {{ .Values.resources.requests.mem }} + imagePullSecrets: + - name: {{ .Values.imagePullSecretsName }} \ No newline at end of file diff --git a/examples/multi-service-demo/full-charts/service2/templates/service.yaml b/examples/multi-service-demo/full-charts/service2/templates/service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d21ce064c5f44a9994b4fe592126e5aff154fa0d --- /dev/null +++ b/examples/multi-service-demo/full-charts/service2/templates/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.fullnameOverride }} + labels: + app.kubernetes.io/name: demo + app.kubernetes.io/instance: {{ .Values.fullnameOverride }} +spec: + selector: + app.kubernetes.io/name: demo + app.kubernetes.io/instance: {{ .Values.fullnameOverride }} + type: NodePort + ports: + - protocol: TCP + port: {{ .Values.port }} + targetPort: {{ .Values.port }} \ No newline at end of file diff --git a/examples/multi-service-demo/full-charts/service2/values.yaml b/examples/multi-service-demo/full-charts/service2/values.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62eca550448c370b9e540e0583b2ea6d3470d060 --- /dev/null +++ b/examples/multi-service-demo/full-charts/service2/values.yaml @@ -0,0 +1,18 @@ +fullnameOverride: service2 +replicaCount: 1 +port: 20222 + +imagePullSecretsName: "default-registry-secret" + +image: + repository: "koderover.tencentcloudcr.com/koderover-demo/service2" + tag: "latest" + +resources: + requests: + cpu: 10m + mem: 10Mi + + limits: + cpu: 20m + mem: 20Mi diff --git a/examples/multi-service-demo/full-charts/service3/Chart.yaml b/examples/multi-service-demo/full-charts/service3/Chart.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4059fc12f464de37a4a95ec6f99bf1b081c85c88 --- /dev/null +++ b/examples/multi-service-demo/full-charts/service3/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: service3 +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/examples/multi-service-demo/full-charts/service3/templates/_helpers.tpl b/examples/multi-service-demo/full-charts/service3/templates/_helpers.tpl new file mode 100644 index 0000000000000000000000000000000000000000..2668feb5bc8a25ef24b51c7fd3c4d9a7d1dcadf5 --- /dev/null +++ b/examples/multi-service-demo/full-charts/service3/templates/_helpers.tpl @@ -0,0 +1,63 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "chart.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "chart.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "chart.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "chart.labels" -}} +helm.sh/chart: {{ include "chart.chart" . }} +{{ include "chart.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "chart.selectorLabels" -}} +app.kubernetes.io/name: {{ include "chart.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "chart.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "chart.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/examples/multi-service-demo/full-charts/service3/templates/deployment.yaml b/examples/multi-service-demo/full-charts/service3/templates/deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e441d6b85ac4b9f2a9d9c5e0f0ad05d3fa89a918 --- /dev/null +++ b/examples/multi-service-demo/full-charts/service3/templates/deployment.yaml @@ -0,0 +1,37 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.fullnameOverride }} + labels: + app.kubernetes.io/name: demo + app.kubernetes.io/instance: {{ .Values.fullnameOverride }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: demo + app.kubernetes.io/instance: {{ .Values.fullnameOverride }} + replicas: {{ .Values.replicaCount }} + template: + metadata: + labels: + app.kubernetes.io/name: demo + app.kubernetes.io/instance: {{ .Values.fullnameOverride }} + spec: + containers: + - name: {{ .Values.fullnameOverride }} + image: {{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: Always + command: + - /workspace/{{ .Values.fullnameOverride }} + ports: + - protocol: TCP + containerPort: {{ .Values.port }} + resources: + limits: + cpu: {{ .Values.resources.limits.cpu }} + memory: {{ .Values.resources.limits.mem }} + requests: + cpu: {{ .Values.resources.requests.cpu }} + memory: {{ .Values.resources.requests.mem }} + imagePullSecrets: + - name: {{ .Values.imagePullSecretsName }} \ No newline at end of file diff --git a/examples/multi-service-demo/full-charts/service3/templates/service.yaml b/examples/multi-service-demo/full-charts/service3/templates/service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d21ce064c5f44a9994b4fe592126e5aff154fa0d --- /dev/null +++ b/examples/multi-service-demo/full-charts/service3/templates/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.fullnameOverride }} + labels: + app.kubernetes.io/name: demo + app.kubernetes.io/instance: {{ .Values.fullnameOverride }} +spec: + selector: + app.kubernetes.io/name: demo + app.kubernetes.io/instance: {{ .Values.fullnameOverride }} + type: NodePort + ports: + - protocol: TCP + port: {{ .Values.port }} + targetPort: {{ .Values.port }} \ No newline at end of file diff --git a/examples/multi-service-demo/full-charts/service3/values.yaml b/examples/multi-service-demo/full-charts/service3/values.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c0c508dc9911664ef5f6f5058e1aa7401568149 --- /dev/null +++ b/examples/multi-service-demo/full-charts/service3/values.yaml @@ -0,0 +1,20 @@ +fullnameOverride: service1 +replicaCount: 1 +port: 20223 + +imagePullSecretsName: "default-registry-secret" + +image: + repository: "koderover.tencentcloudcr.com/koderover-demo/service3" + tag: "latest" + cronjobrepo: "koderover.tencentcloudcr.com/test/busybox" + cronjobtag: "20230620192216" + +resources: + requests: + cpu: 10m + mem: 10Mi + + limits: + cpu: 20m + mem: 20Mi diff --git a/examples/multi-service-demo/general-chart/Chart.yaml b/examples/multi-service-demo/general-chart/Chart.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08e787e5acc8e1c120b74f1e50f7d0ad4a9e1675 --- /dev/null +++ b/examples/multi-service-demo/general-chart/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: general-chart +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/examples/multi-service-demo/general-chart/templates/_helpers.tpl b/examples/multi-service-demo/general-chart/templates/_helpers.tpl new file mode 100644 index 0000000000000000000000000000000000000000..2668feb5bc8a25ef24b51c7fd3c4d9a7d1dcadf5 --- /dev/null +++ b/examples/multi-service-demo/general-chart/templates/_helpers.tpl @@ -0,0 +1,63 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "chart.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "chart.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "chart.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "chart.labels" -}} +helm.sh/chart: {{ include "chart.chart" . }} +{{ include "chart.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "chart.selectorLabels" -}} +app.kubernetes.io/name: {{ include "chart.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "chart.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "chart.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/examples/multi-service-demo/general-chart/templates/deployment.yaml b/examples/multi-service-demo/general-chart/templates/deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ea2a441222d9d37befe183d10124049bb2358a16 --- /dev/null +++ b/examples/multi-service-demo/general-chart/templates/deployment.yaml @@ -0,0 +1,66 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.fullnameOverride }} + labels: + app.kubernetes.io/name: demo + app.kubernetes.io/instance: {{ .Values.fullnameOverride }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: demo + app.kubernetes.io/instance: {{ .Values.fullnameOverride }} + replicas: {{ .Values.replicaCount }} + template: + metadata: + labels: + app.kubernetes.io/name: demo + app.kubernetes.io/instance: {{ .Values.fullnameOverride }} + spec: + volumes: + - name: config + configMap: + name: game-demo + items: + - key: "game.properties" + path: "game.properties" + containers: + - name: {{ .Values.fullnameOverride }} + image: {{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: Always + volumeMounts: + - name: config + mountPath: "/config" + readOnly: true + env: + - name: "x" + value: "x2" + command: + - /workspace/{{ .Values.fullnameOverride }} + ports: + - protocol: TCP + containerPort: {{ .Values.port }} + resources: + limits: + cpu: {{ .Values.resources.limits.cpu }} + memory: {{ .Values.resources.limits.mem }} + requests: + cpu: {{ .Values.resources.requests.cpu }} + memory: {{ .Values.resources.requests.mem }} + imagePullSecrets: + - name: {{ .Values.imagePullSecretsName }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: game-demo +data: + player_initial_lives: "3" + ui_properties_file_name: "user-interface.properties" + game.properties: | + enemy.types=aliens,monsters + player.maximum-lives=5 + user-interface.properties: | + color.good=purple + color.bad=yellow + allow.textmode=true \ No newline at end of file diff --git a/examples/multi-service-demo/general-chart/templates/service.yaml b/examples/multi-service-demo/general-chart/templates/service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d21ce064c5f44a9994b4fe592126e5aff154fa0d --- /dev/null +++ b/examples/multi-service-demo/general-chart/templates/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.fullnameOverride }} + labels: + app.kubernetes.io/name: demo + app.kubernetes.io/instance: {{ .Values.fullnameOverride }} +spec: + selector: + app.kubernetes.io/name: demo + app.kubernetes.io/instance: {{ .Values.fullnameOverride }} + type: NodePort + ports: + - protocol: TCP + port: {{ .Values.port }} + targetPort: {{ .Values.port }} \ No newline at end of file diff --git a/examples/multi-service-demo/general-chart/values.yaml b/examples/multi-service-demo/general-chart/values.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f4247c21edd0f5f12e61bdf4780d5c6e2db5122e --- /dev/null +++ b/examples/multi-service-demo/general-chart/values.yaml @@ -0,0 +1,18 @@ +fullnameOverride: $T-Service$ +replicaCount: 1 +port: {{.port}} + +imagePullSecretsName: "default-registry-secret" + +image: + repository: "koderover.tencentcloudcr.com/koderover-demo/$T-Service$" + tag: "latest" + +resources: + requests: + cpu: 10m + mem: 10Mi + + limits: + cpu: 20m + mem: 20Mi diff --git a/examples/multi-service-demo/go.mod b/examples/multi-service-demo/go.mod new file mode 100644 index 0000000000000000000000000000000000000000..9a50caa184a89afc0c7731f6a81b39da71b8928e --- /dev/null +++ b/examples/multi-service-demo/go.mod @@ -0,0 +1,3 @@ +module multi-service-demo + +go 1.17 diff --git a/examples/multi-service-demo/k8s-yaml/service1/deployment.yaml b/examples/multi-service-demo/k8s-yaml/service1/deployment.yaml index 7eb132f77fdcff713ee9a2247cc38b029090d91d..027cf31c664786f5fd66388d511b1d4d5813d48b 100644 --- a/examples/multi-service-demo/k8s-yaml/service1/deployment.yaml +++ b/examples/multi-service-demo/k8s-yaml/service1/deployment.yaml @@ -19,11 +19,15 @@ spec: spec: containers: - name: service1 - image: ccr.ccs.tencentyun.com/koderover-public/service1:latest + image: koderover.tencentcloudcr.com/koderover-demo/service1:latest imagePullPolicy: Always command: - /workspace/service1 ports: - protocol: TCP - containerPort: 20220 + containerPort: 20221 + resources: + limits: + memory: 50Mi + cpu: 50m \ No newline at end of file diff --git a/examples/multi-service-demo/k8s-yaml/service1/service.yaml b/examples/multi-service-demo/k8s-yaml/service1/service.yaml index e543d7f6ca5af76ca2ea10a39cc2b0a4060b0179..a49ac621ee332b5f4033b7f70f6299fdcb9bc6be 100644 --- a/examples/multi-service-demo/k8s-yaml/service1/service.yaml +++ b/examples/multi-service-demo/k8s-yaml/service1/service.yaml @@ -9,5 +9,5 @@ spec: type: NodePort ports: - protocol: TCP - port: 20220 - targetPort: 20220 \ No newline at end of file + port: 20221 + targetPort: 20221 \ No newline at end of file diff --git a/examples/multi-service-demo/k8s-yaml/service2/deployment.yaml b/examples/multi-service-demo/k8s-yaml/service2/deployment.yaml index b42dfb24f246338a7678b65af378e7b4ff0aa2f6..87a6f842a64abe03ee7bd4014f5dbe87d0b6b806 100644 --- a/examples/multi-service-demo/k8s-yaml/service2/deployment.yaml +++ b/examples/multi-service-demo/k8s-yaml/service2/deployment.yaml @@ -19,11 +19,15 @@ spec: spec: containers: - name: service2 - image: ccr.ccs.tencentyun.com/koderover-public/service2:latest + image: koderover.tencentcloudcr.com/koderover-demo/service2:latest imagePullPolicy: Always command: - /workspace/service2 ports: - protocol: TCP - containerPort: 20220 + containerPort: 20222 + resources: + limits: + memory: 50Mi + cpu: 50m \ No newline at end of file diff --git a/examples/multi-service-demo/k8s-yaml/service2/service.yaml b/examples/multi-service-demo/k8s-yaml/service2/service.yaml index 78b5422b4b080319690a197d18d5e424a2a98137..8ca11162a61a199166ba23e86528784b94e5ca4d 100644 --- a/examples/multi-service-demo/k8s-yaml/service2/service.yaml +++ b/examples/multi-service-demo/k8s-yaml/service2/service.yaml @@ -9,5 +9,5 @@ spec: type: NodePort ports: - protocol: TCP - port: 20221 - targetPort: 20221 \ No newline at end of file + port: 20222 + targetPort: 20222 \ No newline at end of file diff --git a/examples/multi-service-demo/k8s-yaml/service3/deployment.yaml b/examples/multi-service-demo/k8s-yaml/service3/deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d8c6c855e2626841755ece90dc3a96f8d5457ecb --- /dev/null +++ b/examples/multi-service-demo/k8s-yaml/service3/deployment.yaml @@ -0,0 +1,33 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: service3 + labels: + app.kubernetes.io/name: demo + app.kubernetes.io/instance: service3 +spec: + selector: + matchLabels: + app.kubernetes.io/name: demo + app.kubernetes.io/instance: service3 + replicas: 1 + template: + metadata: + labels: + app.kubernetes.io/name: demo + app.kubernetes.io/instance: service3 + spec: + containers: + - name: service3 + image: koderover.tencentcloudcr.com/koderover-demo/service3:latest + imagePullPolicy: Always + command: + - /workspace/service3 + ports: + - protocol: TCP + containerPort: 20223 + resources: + limits: + memory: 100Mi + cpu: 50m + \ No newline at end of file diff --git a/examples/multi-service-demo/k8s-yaml/service3/service.yaml b/examples/multi-service-demo/k8s-yaml/service3/service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c36bac62729d063713dd9a836d519ed4439cc802 --- /dev/null +++ b/examples/multi-service-demo/k8s-yaml/service3/service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: service3 + labels: + app.kubernetes.io/name: demo + app.kubernetes.io/instance: service3 +spec: + type: NodePort + ports: + - protocol: TCP + port: 20223 + targetPort: 20223 \ No newline at end of file diff --git a/examples/multi-service-demo/k8s-yaml/template.yaml b/examples/multi-service-demo/k8s-yaml/template.yaml new file mode 100644 index 0000000000000000000000000000000000000000..520becd8d25a5da9102f32774b89914a486db52c --- /dev/null +++ b/examples/multi-service-demo/k8s-yaml/template.yaml @@ -0,0 +1,46 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: $T-Service$ + labels: + app.kubernetes.io/name: $T-Project$ + app.kubernetes.io/instance: $T-Service$ +spec: + selector: + matchLabels: + app.kubernetes.io/name: $T-Project$ + app.kubernetes.io/instance: $T-Service$ + replicas: 2 + template: + metadata: + labels: + app.kubernetes.io/name: $T-Project$ + app.kubernetes.io/instance: $T-Service$ + spec: + containers: + - name: $T-Service$ + image: koderover.tencentcloudcr.com/koderover-demo/$T-Service$:latest + imagePullPolicy: Always + command: + - /workspace/$T-Service$ + ports: + - protocol: TCP + containerPort: {{.port}} + resources: + limits: + memory: {{.memoryLimit}} + cpu: {{.cpuLimit}} +--- +apiVersion: v1 +kind: Service +metadata: + name: $T-Service$ + labels: + app.kubernetes.io/name: $T-Project$ + app.kubernetes.io/instance: $T-Service$ +spec: + type: NodePort + ports: + - protocol: TCP + port: {{.port}} + targetPort: {{.port}} \ No newline at end of file diff --git a/examples/multi-service-demo/service1/service1.go b/examples/multi-service-demo/src/service1/service1.go similarity index 91% rename from examples/multi-service-demo/service1/service1.go rename to examples/multi-service-demo/src/service1/service1.go index d8e0540e3c56a6f5addc56c12028a5b3b2614fd0..2d3fdcc1ca1f1bc48763a68e59150017b28bfa48 100644 --- a/examples/multi-service-demo/service1/service1.go +++ b/examples/multi-service-demo/src/service1/service1.go @@ -23,7 +23,7 @@ import ( func hello(w http.ResponseWriter, req *http.Request) { - fmt.Fprintf(w, "hello, my name is service1~~\n") + fmt.Fprintf(w, "hello, my name is service1~\n") } func headers(w http.ResponseWriter, req *http.Request) { @@ -40,5 +40,5 @@ func main() { http.HandleFunc("/", hello) http.HandleFunc("/headers", headers) - http.ListenAndServe(":20220", nil) + http.ListenAndServe(":20222", nil) } diff --git a/examples/multi-service-demo/service2/service2.go b/examples/multi-service-demo/src/service2/service2.go similarity index 100% rename from examples/multi-service-demo/service2/service2.go rename to examples/multi-service-demo/src/service2/service2.go diff --git a/examples/multi-service-demo/src/service3/service3.go b/examples/multi-service-demo/src/service3/service3.go new file mode 100644 index 0000000000000000000000000000000000000000..f88096cf2f39eda17ba23197b00158190c55d104 --- /dev/null +++ b/examples/multi-service-demo/src/service3/service3.go @@ -0,0 +1,44 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "net/http" +) + +func hello(w http.ResponseWriter, req *http.Request) { + + fmt.Fprintf(w, "hello, my name is service3~~\n") +} + +func headers(w http.ResponseWriter, req *http.Request) { + + for name, headers := range req.Header { + for _, h := range headers { + fmt.Fprintf(w, "%v: %v\n", name, h) + } + } +} + +func main() { + + http.HandleFunc("/", hello) + http.HandleFunc("/headers", headers) + + http.ListenAndServe(":20223", nil) +} diff --git a/examples/multi-service-demo/charts/service1/values.yaml b/examples/multi-service-demo/values/service1.yaml similarity index 41% rename from examples/multi-service-demo/charts/service1/values.yaml rename to examples/multi-service-demo/values/service1.yaml index 5d7aa51e937827813a66e21415e188eff65feadf..e069766a96a8f16140eee86faf9c1c0ec29ad0c8 100644 --- a/examples/multi-service-demo/charts/service1/values.yaml +++ b/examples/multi-service-demo/values/service1.yaml @@ -1,22 +1,18 @@ - - fullnameOverride: "service1" replicaCount: 1 -port: 20220 +port: 20221 -registrySecret: "" -imagePullSecrets: "service1-secret" +imagePullSecretsName: "default-registry-secret" image: - repository: "ccr.ccs.tencentyun.com/koderover-public/service1" + repository: "koderover.tencentcloudcr.com/koderover-demo/service1" tag: "latest" - resources: requests: - cpu: 1 - mem: 512Mi - - limit: cpu: 10m mem: 10Mi + + limits: + cpu: 20m + mem: 20Mi diff --git a/examples/multi-service-demo/charts/service2/values.yaml b/examples/multi-service-demo/values/service2.yaml similarity index 45% rename from examples/multi-service-demo/charts/service2/values.yaml rename to examples/multi-service-demo/values/service2.yaml index 0a4f4cac19b549e2db051a5dd24beae2e2e5ea9c..4a7e61118f41d4f745ce4ace8cf9d93b8bc97f36 100644 --- a/examples/multi-service-demo/charts/service2/values.yaml +++ b/examples/multi-service-demo/values/service2.yaml @@ -1,22 +1,18 @@ - - fullnameOverride: "service2" replicaCount: 1 -port: 20221 +port: 20222 -registrySecret: "" -imagePullSecrets: "service2-secret" +imagePullSecretsName: "default-registry-secret" image: - repository: "ccr.ccs.tencentyun.com/koderover-public/service2" + repository: "koderover.tencentcloudcr.com/koderover-demo/service2" tag: "latest" - resources: requests: - cpu: 1 - mem: 512Mi - - limits: cpu: 10m mem: 10Mi + + limits: + cpu: 20m + mem: 20Mi diff --git a/examples/multi-service-demo/values/service3.yaml b/examples/multi-service-demo/values/service3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f01e84fc29a05e864815a75c14e9a03f79349d51 --- /dev/null +++ b/examples/multi-service-demo/values/service3.yaml @@ -0,0 +1,19 @@ +fullnameOverride: "service3" +replicaCount: 1 +port: 20223 + +imagePullSecretsName: "default-registry-secret" + +image: + repository: "koderover.tencentcloudcr.com/koderover-demo/service3" + tag: "latest" + +resources: + requests: + cpu: 10m + mem: 10Mi + + limits: + cpu: 20m + mem: 20Mi + diff --git a/examples/nginx/Dockerfile b/examples/nginx/Dockerfile index 7fea54fd2f6e6e65926d635a2c4376244c608670..9d06efcef09ebdbe81562ac4c4e5d096b7627689 100644 --- a/examples/nginx/Dockerfile +++ b/examples/nginx/Dockerfile @@ -1,2 +1,2 @@ -from ccr.ccs.tencentyun.com/shuhe/nginx:stable +from koderover.tencentcloudcr.com/koderover-demo/nginx add ./index.html /usr/share/nginx/html \ No newline at end of file diff --git a/examples/nginx/yaml/nginx.yaml b/examples/nginx/yaml/nginx.yaml index b37915ec415f6a0dfffd49a4a29899b191521102..3a770e90ab45099cc7eb0485642bd7b82266bba7 100644 --- a/examples/nginx/yaml/nginx.yaml +++ b/examples/nginx/yaml/nginx.yaml @@ -35,18 +35,22 @@ spec: spec: containers: - name: nginx-test - image: ccr.ccs.tencentyun.com/shuhe/nginx:stable + image: koderover.tencentcloudcr.com/koderover-demo/nginx ports: - containerPort: 80 volumeMounts: - name: static-page mountPath: /usr/share/nginx/html + resources: + limits: + memory: 50Mi + cpu: 50m volumes: - name: static-page configMap: name: static-page --- -apiVersion: extensions/v1beta1 +apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: nginx-expose @@ -56,9 +60,12 @@ spec: http: paths: - backend: - serviceName: nginx - servicePort: 80 + service: + name: nginx + port: + number: 80 path: / + pathType: Prefix --- apiVersion: v1 kind: ConfigMap diff --git a/examples/nginx/yaml/redis/redis.yaml b/examples/nginx/yaml/redis/redis.yaml index f63f07cb37b53dfef975136bad93ac1b7b00fe02..3d3f238dfa070be217dbaecb242ae7d9c75ab228 100644 --- a/examples/nginx/yaml/redis/redis.yaml +++ b/examples/nginx/yaml/redis/redis.yaml @@ -22,7 +22,7 @@ spec: - name: master image: k8s.gcr.io/redis:e2e # or just image: redis resources: - requests: + limits: cpu: 100m memory: 100Mi ports: diff --git a/examples/pytest-demo/fail_test.py b/examples/pytest-demo/fail_test.py new file mode 100644 index 0000000000000000000000000000000000000000..dc494b1a7707ae6577aba31e95d0ed6ec729d6bd --- /dev/null +++ b/examples/pytest-demo/fail_test.py @@ -0,0 +1,5 @@ +def square(x): + return x * x + +def test_the_square_of_negative_5_is_negative_25(): + assert square(-5) == -25 diff --git a/examples/pytest-demo/pass_test.py b/examples/pytest-demo/pass_test.py new file mode 100644 index 0000000000000000000000000000000000000000..64f0271c0dc1ce290e5d1479ecbbda62d8df290d --- /dev/null +++ b/examples/pytest-demo/pass_test.py @@ -0,0 +1,11 @@ +def increase(x): + return x + 1 + +def test_increase_1_to_2021_is_2022(): + assert increase(2021) == 2022 + +def square(x): + return x * x + +def test_the_square_of_5_is_25(): + assert square(5) == 25 diff --git a/examples/simple-service/nginx-yaml/nginx.yaml b/examples/simple-service/nginx-yaml/nginx.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8d573c027968a164f59096d3d45655e89ccf8ce --- /dev/null +++ b/examples/simple-service/nginx-yaml/nginx.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: Service +metadata: + name: nginx +spec: + ports: + - name: http + port: 80 + targetPort: 80 + selector: + app: nginx + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx +spec: + selector: + matchLabels: + app: nginx + replicas: 1 + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: koderover.tencentcloudcr.com/koderover-demo/nginx:v1 + ports: + - containerPort: 80 + resources: + limits: + cpu: 100m + memory: 100Mi diff --git a/examples/simple-service/service-yaml/a.yaml b/examples/simple-service/service-yaml/a.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2e8ef5c4d396fd78ba67fbcc4dae034102f8237 --- /dev/null +++ b/examples/simple-service/service-yaml/a.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: Service +metadata: + name: a + labels: + app: a +spec: + ports: + - name: http + port: 80 + targetPort: 8080 + selector: + app: a + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: a + labels: + app: a +spec: + selector: + matchLabels: + app: a + replicas: 1 + template: + metadata: + labels: + app: a + spec: + containers: + - name: myapp-1 + image: koderover.tencentcloudcr.com/koderover-demo/myapp-1:v0.1__linux_amd64 + imagePullPolicy: Always + command: ["/myapp-1"] + args: ["--downstream-addr", "$(DOWNSTREAM_ADDR)", "--headers", "$(HEADERS)"] + env: + - name: DOWNSTREAM_ADDR + value: "b" + - name: HEADERS + value: "x-request-id" + ports: + - containerPort: 8080 + resources: + limits: + cpu: 100m + memory: 100Mi diff --git a/examples/simple-service/service-yaml/b.yaml b/examples/simple-service/service-yaml/b.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e5ffb2026fb0fe3bedc010122a46b16e77b2bc3 --- /dev/null +++ b/examples/simple-service/service-yaml/b.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: Service +metadata: + name: b + labels: + app: b +spec: + ports: + - name: http + port: 80 + targetPort: 8081 + selector: + app: b + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: b + labels: + app: b +spec: + selector: + matchLabels: + app: b + replicas: 1 + template: + metadata: + labels: + app: b + spec: + containers: + - name: myapp-2 + image: koderover.tencentcloudcr.com/koderover-demo/myapp-2:v0.1__linux_amd64 + imagePullPolicy: Always + command: ["/myapp-2"] + args: ["--downstream-addr", "$(DOWNSTREAM_ADDR)", "--headers", "$(HEADERS)"] + env: + - name: DOWNSTREAM_ADDR + value: "c" + - name: HEADERS + value: "x-request-id" + ports: + - containerPort: 8081 + resources: + limits: + cpu: 100m + memory: 100Mi diff --git a/examples/simple-service/service-yaml/c.yaml b/examples/simple-service/service-yaml/c.yaml new file mode 100644 index 0000000000000000000000000000000000000000..60c751ba8f08f15c4d1999591eb1ac3c38b565e8 --- /dev/null +++ b/examples/simple-service/service-yaml/c.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: Service +metadata: + name: c + labels: + app: c +spec: + ports: + - name: http + port: 80 + targetPort: 8082 + selector: + app: c + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: c + labels: + app: c +spec: + selector: + matchLabels: + app: c + replicas: 1 + template: + metadata: + labels: + app: c + spec: + containers: + - name: myapp-3 + image: koderover.tencentcloudcr.com/koderover-demo/myapp-3:v0.1__linux_amd64 + imagePullPolicy: Always + command: ["/myapp-3"] + ports: + - containerPort: 8082 + resources: + limits: + cpu: 100m + memory: 100Mi diff --git a/examples/spring-boot-acme-financial/acme-financial-account.yaml b/examples/spring-boot-acme-financial/acme-financial-account.yaml new file mode 100644 index 0000000000000000000000000000000000000000..04c662d53604019c8a9c2f457679ebb088ab38ee --- /dev/null +++ b/examples/spring-boot-acme-financial/acme-financial-account.yaml @@ -0,0 +1,78 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: acme-financial-account + name: acme-financial-account +spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: acme-financial-account + template: + metadata: + labels: + app: acme-financial-account + spec: + initContainers: + - name: init-skywalking-agent + image: koderover.tencentcloudcr.com/koderover-demo/skywalking-agent:8.8.0 + command: + - 'sh' + - '-c' + - 'set -ex;mkdir -p /vmskywalking/agent;cp -r /usr/skywalking/agent/* /vmskywalking/agent;' + volumeMounts: + - mountPath: /vmskywalking/agent + name: skywalking-agent + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 100m + memory: 256Mi + containers: + - env: + - name: JAVA_OPTS + value: "-javaagent:/opt/skywalking/agent/skywalking-agent.jar" + - name: SW_AGENT_NAME + value: "acme-financial-account" + - name: SW_AGENT_COLLECTOR_BACKEND_SERVICES + value: "skywalking-oap.skywalking.svc.cluster.local:11800" + image: koderover.tencentcloudcr.com/koderover-demo/acme-financial-account:v0.1 + imagePullPolicy: Always + name: account + ports: + - containerPort: 8083 + protocol: TCP + resources: + limits: + cpu: 1000m + memory: 2Gi + requests: + cpu: 100m + memory: 512Mi + volumeMounts: + - mountPath: /opt/skywalking/agent + name: skywalking-agent + volumes: + - name: skywalking-agent + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: acme-financial-account + name: acme-financial-account +spec: + ports: + - name: http + port: 8083 + protocol: TCP + targetPort: 8083 + selector: + app: acme-financial-account + sessionAffinity: None + type: ClusterIP \ No newline at end of file diff --git a/examples/spring-boot-acme-financial/acme-financial-customer.yaml b/examples/spring-boot-acme-financial/acme-financial-customer.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2bba4aa8a9c4f27912ac207537e1ff3391c3e6cf --- /dev/null +++ b/examples/spring-boot-acme-financial/acme-financial-customer.yaml @@ -0,0 +1,77 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: acme-financial-customer + name: acme-financial-customer +spec: + replicas: 1 + selector: + matchLabels: + app: acme-financial-customer + template: + metadata: + labels: + app: acme-financial-customer + spec: + initContainers: + - name: init-skywalking-agent + image: koderover.tencentcloudcr.com/koderover-demo/skywalking-agent:8.8.0 + command: + - 'sh' + - '-c' + - 'set -ex;mkdir -p /vmskywalking/agent;cp -r /usr/skywalking/agent/* /vmskywalking/agent;' + volumeMounts: + - mountPath: /vmskywalking/agent + name: skywalking-agent + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 100m + memory: 256Mi + containers: + - env: + - name: JAVA_OPTS + value: "-javaagent:/opt/skywalking/agent/skywalking-agent.jar" + - name: SW_AGENT_NAME + value: "acme-financial-customer" + - name: SW_AGENT_COLLECTOR_BACKEND_SERVICES + value: "skywalking-oap.skywalking.svc.cluster.local:11800" + image: koderover.tencentcloudcr.com/koderover-demo/acme-financial-customer:v0.1 + imagePullPolicy: Always + name: customer + ports: + - containerPort: 8084 + protocol: TCP + volumeMounts: + - mountPath: /opt/skywalking/agent + name: skywalking-agent + resources: + limits: + cpu: 1000m + memory: 2Gi + requests: + cpu: 100m + memory: 512Mi + volumes: + - name: skywalking-agent + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: acme-financial-customer + name: acme-financial-customer +spec: + ports: + - name: http + port: 8084 + protocol: TCP + targetPort: 8084 + selector: + app: acme-financial-customer + sessionAffinity: None + type: ClusterIP \ No newline at end of file diff --git a/examples/spring-boot-acme-financial/acme-financial-office.yaml b/examples/spring-boot-acme-financial/acme-financial-office.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c96e7fa7191902939c11dde9636be2f770a14d1 --- /dev/null +++ b/examples/spring-boot-acme-financial/acme-financial-office.yaml @@ -0,0 +1,77 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: acme-financial-office + name: acme-financial-office +spec: + replicas: 1 + selector: + matchLabels: + app: acme-financial-office + template: + metadata: + labels: + app: acme-financial-office + spec: + initContainers: + - name: init-skywalking-agent + image: koderover.tencentcloudcr.com/koderover-demo/skywalking-agent:8.8.0 + command: + - 'sh' + - '-c' + - 'set -ex;mkdir -p /vmskywalking/agent;cp -r /usr/skywalking/agent/* /vmskywalking/agent;' + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 100m + memory: 256Mi + volumeMounts: + - mountPath: /vmskywalking/agent + name: skywalking-agent + containers: + - env: + - name: JAVA_OPTS + value: "-javaagent:/opt/skywalking/agent/skywalking-agent.jar" + - name: SW_AGENT_NAME + value: "acme-financial-office" + - name: SW_AGENT_COLLECTOR_BACKEND_SERVICES + value: "skywalking-oap.skywalking.svc.cluster.local:11800" + image: koderover.tencentcloudcr.com/koderover-demo/acme-financial-office:v0.1 + imagePullPolicy: Always + name: office + ports: + - containerPort: 8082 + protocol: TCP + resources: + limits: + cpu: 1000m + memory: 2Gi + requests: + cpu: 100m + memory: 512Mi + volumeMounts: + - mountPath: /opt/skywalking/agent + name: skywalking-agent + volumes: + - name: skywalking-agent + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: acme-financial-office + name: acme-financial-back-office +spec: + ports: + - name: http + port: 8082 + protocol: TCP + targetPort: 8082 + selector: + app: acme-financial-office + sessionAffinity: None + type: ClusterIP \ No newline at end of file diff --git a/examples/spring-boot-acme-financial/acme-financial-ui.yaml b/examples/spring-boot-acme-financial/acme-financial-ui.yaml new file mode 100644 index 0000000000000000000000000000000000000000..58ccd712a32c3a81a52c9adecbe3653132c7647d --- /dev/null +++ b/examples/spring-boot-acme-financial/acme-financial-ui.yaml @@ -0,0 +1,77 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: acme-financial-ui + name: acme-financial-ui +spec: + replicas: 1 + selector: + matchLabels: + app: acme-financial-ui + template: + metadata: + labels: + app: acme-financial-ui + spec: + initContainers: + - name: init-skywalking-agent + image: koderover.tencentcloudcr.com/koderover-demo/skywalking-agent:8.8.0 + command: + - 'sh' + - '-c' + - 'set -ex;mkdir -p /vmskywalking/agent;cp -r /usr/skywalking/agent/* /vmskywalking/agent;' + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 100m + memory: 256Mi + volumeMounts: + - mountPath: /vmskywalking/agent + name: skywalking-agent + containers: + - env: + - name: JAVA_OPTS + value: "-javaagent:/opt/skywalking/agent/skywalking-agent.jar" + - name: SW_AGENT_NAME + value: "acme-financial-ui" + - name: SW_AGENT_COLLECTOR_BACKEND_SERVICES + value: "skywalking-oap.skywalking.svc.cluster.local:11800" + image: koderover.tencentcloudcr.com/koderover-demo/acme-financial-ui:v0.1 + imagePullPolicy: Always + name: ui + ports: + - containerPort: 8081 + protocol: TCP + resources: + limits: + cpu: 1000m + memory: 2Gi + requests: + cpu: 100m + memory: 512Mi + volumeMounts: + - mountPath: /opt/skywalking/agent + name: skywalking-agent + volumes: + - name: skywalking-agent + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: acme-financial-ui + name: acme-financial-ui +spec: + ports: + - name: http + port: 8081 + protocol: TCP + targetPort: 8081 + selector: + app: acme-financial-ui + sessionAffinity: None + type: NodePort \ No newline at end of file diff --git a/examples/spring-cloud-piggymetrics/account-mongodb-service/7-account-mongodb-service.yaml b/examples/spring-cloud-piggymetrics/account-mongodb-service/7-account-mongodb-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..90d62cb32c2498a97bf3472b128b582fdc6008ff --- /dev/null +++ b/examples/spring-cloud-piggymetrics/account-mongodb-service/7-account-mongodb-service.yaml @@ -0,0 +1,63 @@ +apiVersion: v1 +kind: Service +metadata: + name: account-mongodb + labels: + project: piggymetrics + tier: database + app: account-mongodb +spec: + ports: + - name: mongo + port: 27017 + targetPort: 27017 + selector: + project: piggymetrics + tier: database + app: account-mongodb +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: account-mongodb + labels: + project: piggymetrics + tier: database + app: account-mongodb +spec: + replicas: 1 + selector: + matchLabels: + project: piggymetrics + tier: database + app: account-mongodb + strategy: + type: Recreate + template: + metadata: + labels: + project: piggymetrics + tier: database + app: account-mongodb + spec: + containers: + - name: account-mongodb + env: + - name: INIT_DUMP + value: account-service-dump.js + - name: MONGODB_PASSWORD + valueFrom: + secretKeyRef: + name: piggymetrics + key: account_mongodb_password + ports: + - containerPort: 27017 + image: koderover.tencentcloudcr.com/koderover-demo/piggymetrics-mongodb:v0.1.1 + resources: + limits: + cpu: 500m + memory: 1G + requests: + cpu: 100m + memory: 256M + restartPolicy: Always diff --git a/examples/spring-cloud-piggymetrics/account-service/8-account-service.yaml b/examples/spring-cloud-piggymetrics/account-service/8-account-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a6477665864d710f05bbf1801b68d632c2711435 --- /dev/null +++ b/examples/spring-cloud-piggymetrics/account-service/8-account-service.yaml @@ -0,0 +1,97 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + project: piggymetrics + tier: backend + app: account-service + name: account-service +spec: + ports: + - name: http + port: 6000 + targetPort: 6000 + selector: + project: piggymetrics + tier: backend + app: account-service +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + project: piggymetrics + tier: backend + app: account-service + name: account-service +spec: + replicas: 1 + selector: + matchLabels: + project: piggymetrics + tier: backend + app: account-service + strategy: + type: RollingUpdate + template: + metadata: + labels: + project: piggymetrics + tier: backend + app: account-service + spec: + initContainers: + - name: init-skywalking-agent + image: koderover.tencentcloudcr.com/koderover-demo/skywalking-agent:8.8.0 + command: + - 'sh' + - '-c' + - 'set -ex;mkdir -p /vmskywalking/agent;cp -r /usr/skywalking/agent/* /vmskywalking/agent;' + volumeMounts: + - mountPath: /vmskywalking/agent + name: skywalking-agent + containers: + - env: + - name: JAVA_OPTS + value: "-javaagent:/opt/skywalking/agent/skywalking-agent.jar -Duser.timezone=GMT+8" + - name: SW_AGENT_NAME + value: "account-service" + - name: SW_AGENT_COLLECTOR_BACKEND_SERVICES + value: "skywalking-oap.skywalking.svc.cluster.local:11800" + - name: ACCOUNT_SERVICE_PASSWORD + valueFrom: + secretKeyRef: + name: piggymetrics + key: account_service_password + - name: CONFIG_SERVICE_PASSWORD + valueFrom: + secretKeyRef: + name: piggymetrics + key: config_service_password + - name: MONGODB_PASSWORD + valueFrom: + secretKeyRef: + name: piggymetrics + key: account_mongodb_password + envFrom: + - configMapRef: + name: special-config-env + ports: + - containerPort: 6000 + image: koderover.tencentcloudcr.com/koderover-demo/piggymetrics-account-service:v0.3.0 + name: account-service + resources: + limits: + cpu: "1" + memory: 1G + requests: + cpu: 500m + memory: 512M + imagePullPolicy: Always + volumeMounts: + - mountPath: /opt/skywalking/agent + name: skywalking-agent + restartPolicy: Always + volumes: + - name: skywalking-agent + emptyDir: {} diff --git a/examples/spring-cloud-piggymetrics/auth-mongodb-service/5-auth-mongodb-service.yaml b/examples/spring-cloud-piggymetrics/auth-mongodb-service/5-auth-mongodb-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..619915f0e79bc1289f646dbc7f137a2d65ff472c --- /dev/null +++ b/examples/spring-cloud-piggymetrics/auth-mongodb-service/5-auth-mongodb-service.yaml @@ -0,0 +1,61 @@ +apiVersion: v1 +kind: Service +metadata: + name: auth-mongodb + labels: + project: piggymetrics + tier: database + app: auth-mongodb +spec: + ports: + - name: mongo + port: 27017 + targetPort: 27017 + selector: + project: piggymetrics + tier: database + app: auth-mongodb +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: auth-mongodb + labels: + project: piggymetrics + tier: database + app: auth-mongodb +spec: + replicas: 1 + selector: + matchLabels: + project: piggymetrics + tier: database + app: auth-mongodb + strategy: + type: Recreate + template: + metadata: + labels: + project: piggymetrics + tier: database + app: auth-mongodb + spec: + containers: + - image: koderover.tencentcloudcr.com/koderover-demo/piggymetrics-mongodb:v0.1.1 + name: auth-mongodb + env: + - name: MONGODB_PASSWORD + valueFrom: + secretKeyRef: + name: piggymetrics + key: auth_mongodb_password + ports: + - containerPort: 27017 + resources: + limits: + cpu: 500m + memory: 1G + requests: + cpu: 100m + memory: 256M + restartPolicy: Always diff --git a/examples/spring-cloud-piggymetrics/auth-service/6-auth-service.yaml b/examples/spring-cloud-piggymetrics/auth-service/6-auth-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d8fdde8679ee6eb0bc83d7469d30b4d6eaefdcac --- /dev/null +++ b/examples/spring-cloud-piggymetrics/auth-service/6-auth-service.yaml @@ -0,0 +1,107 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + project: piggymetrics + tier: infrastructure + app: auth-service + name: auth-service +spec: + ports: + - name: http + port: 5000 + targetPort: 5000 + selector: + project: piggymetrics + tier: infrastructure + app: auth-service +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + project: piggymetrics + tier: infrastructure + app: auth-service + name: auth-service +spec: + replicas: 1 + selector: + matchLabels: + project: piggymetrics + tier: infrastructure + app: auth-service + strategy: + type: RollingUpdate + template: + metadata: + labels: + project: piggymetrics + tier: infrastructure + app: auth-service + spec: + initContainers: + - name: init-skywalking-agent + image: koderover.tencentcloudcr.com/koderover-demo/skywalking-agent:8.8.0 + command: + - 'sh' + - '-c' + - 'set -ex;mkdir -p /vmskywalking/agent;cp -r /usr/skywalking/agent/* /vmskywalking/agent;' + volumeMounts: + - mountPath: /vmskywalking/agent + name: skywalking-agent + containers: + - env: + - name: JAVA_OPTS + value: "-javaagent:/opt/skywalking/agent/skywalking-agent.jar -Duser.timezone=GMT+8" + - name: SW_AGENT_NAME + value: "auth-service" + - name: SW_AGENT_COLLECTOR_BACKEND_SERVICES + value: "skywalking-oap.skywalking.svc.cluster.local:11800" + - name: ACCOUNT_SERVICE_PASSWORD + valueFrom: + secretKeyRef: + name: piggymetrics + key: account_service_password + - name: CONFIG_SERVICE_PASSWORD + valueFrom: + secretKeyRef: + name: piggymetrics + key: config_service_password + - name: MONGODB_PASSWORD + valueFrom: + secretKeyRef: + name: piggymetrics + key: auth_mongodb_password + - name: NOTIFICATION_SERVICE_PASSWORD + valueFrom: + secretKeyRef: + name: piggymetrics + key: notification_service_password + - name: STATISTICS_SERVICE_PASSWORD + valueFrom: + secretKeyRef: + name: piggymetrics + key: statistics_service_password + envFrom: + - configMapRef: + name: special-config-env + image: koderover.tencentcloudcr.com/koderover-demo/piggymetrics-auth-service:v0.3.0 + name: auth-service + resources: + limits: + cpu: "1" + memory: 1G + requests: + cpu: 500m + memory: 512M + imagePullPolicy: Always + ports: + - containerPort: 5000 + volumeMounts: + - mountPath: /opt/skywalking/agent + name: skywalking-agent + restartPolicy: Always + volumes: + - name: skywalking-agent + emptyDir: {} diff --git a/examples/spring-cloud-piggymetrics/config-service/2-config-service.yaml b/examples/spring-cloud-piggymetrics/config-service/2-config-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..279d0144fec94788fb6dfe72c869395667ccd322 --- /dev/null +++ b/examples/spring-cloud-piggymetrics/config-service/2-config-service.yaml @@ -0,0 +1,119 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + project: piggymetrics + tier: infrastructure + app: config + name: config +spec: + ports: + - name: http + port: 8888 + targetPort: 8888 + selector: + project: piggymetrics + tier: infrastructure + app: config +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + project: piggymetrics + tier: infrastructure + app: config + name: config +spec: + replicas: 1 + selector: + matchLabels: + project: piggymetrics + tier: infrastructure + app: config + strategy: + type: RollingUpdate + template: + metadata: + labels: + project: piggymetrics + tier: infrastructure + app: config + spec: + initContainers: + - name: init-skywalking-agent + image: koderover.tencentcloudcr.com/koderover-demo/skywalking-agent:8.8.0 + command: + - 'sh' + - '-c' + - 'set -ex;mkdir -p /vmskywalking/agent;cp -r /usr/skywalking/agent/* /vmskywalking/agent;' + volumeMounts: + - mountPath: /vmskywalking/agent + name: skywalking-agent + containers: + - env: + - name: JAVA_OPTS + value: "-javaagent:/opt/skywalking/agent/skywalking-agent.jar -Duser.timezone=GMT+8" + - name: SW_AGENT_NAME + value: "config" + - name: SW_AGENT_COLLECTOR_BACKEND_SERVICES + value: "skywalking-oap.skywalking.svc.cluster.local:11800" + - name: CONFIG_SERVICE_PASSWORD + valueFrom: + secretKeyRef: + name: piggymetrics + key: config_service_password + envFrom: + - configMapRef: + name: special-config-env + image: koderover.tencentcloudcr.com/koderover-demo/piggymetrics-config:v0.3.0 + name: config + resources: + limits: + cpu: 500m + memory: 1G + requests: + cpu: 200m + memory: 512M + imagePullPolicy: Always + ports: + - containerPort: 8888 + volumeMounts: + - mountPath: /opt/skywalking/agent + name: skywalking-agent + restartPolicy: Always + volumes: + - name: skywalking-agent + emptyDir: {} +--- +apiVersion: v1 +kind: Secret +metadata: + name: piggymetrics +type: Opaque +data: + config_service_password: Q09ORklHX1NFUlZJQ0VfUEFTU1dPUkQ= + auth_mongodb_password: QVVUSF9NT05HT0RCX1BBU1NXT1JE + notification_service_password: Tk9USUZJQ0FUSU9OX1NFUlZJQ0VfUEFTU1dPUkQ= + notification_mongodb_password: Tk9USUZJQ0FUSU9OX01PTkdPREJfUEFTU1dPUkQ= + statistics_service_password: U1RBVElTVElDU19TRVJWSUNFX1BBU1NXT1JE + statistics_mongodb_password: U1RBVElTVElDU19NT05HT0RCX1BBU1NXT1JE + account_service_password: QUNDT1VOVF9TRVJWSUNFX1BBU1NXT1JE + account_mongodb_password: QUNDT1VOVF9NT05HT0RCX1BBU1NXT1JE + notification_email_host: c210cC5nbWFpbC5jb20= + notification_email_port: NDY1 + notification_email_user: ZGV2LXVzZXI= + notification_email_pass: ZGV2LXBhc3N3b3Jk +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: special-config-env +data: + registry_service_host: registry + auth_service_host: auth-service + auth_mongodb_host: auth-mongodb + account_mongodb_host: account-mongodb + notification_mongodb_host: notification-mongodb + statistics_mongodb_host: statistics-mongodb + rabbitmq_host: rabbitmq \ No newline at end of file diff --git a/examples/spring-cloud-piggymetrics/gateway-service/4-gateway-service.yaml b/examples/spring-cloud-piggymetrics/gateway-service/4-gateway-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..60bc3d9945058ea8e7bf20e3d4d543073cde88de --- /dev/null +++ b/examples/spring-cloud-piggymetrics/gateway-service/4-gateway-service.yaml @@ -0,0 +1,96 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + project: piggymetrics + tier: frontend + app: gateway + name: gateway +spec: + # comment or delete the following line if you want to use a LoadBalancer + type: NodePort + # if your cluster supports it, uncomment the following to automatically create + # an external load-balanced IP for the frontend service. + # type: LoadBalancer + ports: + - name: http + port: 80 + targetPort: 4000 + # comment or delete the following line if you want to use a LoadBalancer + nodePort: 30080 + selector: + project: piggymetrics + tier: frontend + app: gateway +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gateway + labels: + project: piggymetrics + tier: frontend + app: gateway +spec: + replicas: 1 + selector: + matchLabels: + project: piggymetrics + tier: frontend + app: gateway + strategy: + type: RollingUpdate + template: + metadata: + creationTimestamp: null + labels: + project: piggymetrics + tier: frontend + app: gateway + spec: + initContainers: + - name: init-skywalking-agent + image: koderover.tencentcloudcr.com/koderover-demo/skywalking-agent:8.8.0 + command: + - 'sh' + - '-c' + - 'set -ex;mkdir -p /vmskywalking/agent;cp -r /usr/skywalking/agent/* /vmskywalking/agent;' + volumeMounts: + - mountPath: /vmskywalking/agent + name: skywalking-agent + containers: + - name: gateway + env: + - name: JAVA_OPTS + value: "-javaagent:/opt/skywalking/agent/skywalking-agent.jar -Duser.timezone=GMT+8" + - name: SW_AGENT_NAME + value: "gateway" + - name: SW_AGENT_COLLECTOR_BACKEND_SERVICES + value: "skywalking-oap.skywalking.svc.cluster.local:11800" + - name: CONFIG_SERVICE_PASSWORD + valueFrom: + secretKeyRef: + name: piggymetrics + key: config_service_password + envFrom: + - configMapRef: + name: special-config-env + image: koderover.tencentcloudcr.com/koderover-demo/piggymetrics-gateway:v0.3.0 + name: gateway + resources: + limits: + cpu: "1" + memory: 1G + requests: + cpu: 500m + memory: 512M + imagePullPolicy: Always + ports: + - containerPort: 4000 + volumeMounts: + - mountPath: /opt/skywalking/agent + name: skywalking-agent + restartPolicy: Always + volumes: + - name: skywalking-agent + emptyDir: {} diff --git a/examples/spring-cloud-piggymetrics/monitoring-service/13-monitoring-service.yaml b/examples/spring-cloud-piggymetrics/monitoring-service/13-monitoring-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..df5efd73e8f38955aab3a1a441d3e8cb3a6c81f5 --- /dev/null +++ b/examples/spring-cloud-piggymetrics/monitoring-service/13-monitoring-service.yaml @@ -0,0 +1,89 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + project: piggymetrics + tier: infrastructure + app: monitoring + name: monitoring +spec: + type: NodePort + ports: + - name: http + port: 9000 + targetPort: 8080 + nodePort: 31090 + selector: + project: piggymetrics + tier: infrastructure + app: monitoring +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + project: piggymetrics + tier: infrastructure + app: monitoring + name: monitoring +spec: + replicas: 1 + selector: + matchLabels: + project: piggymetrics + tier: infrastructure + app: monitoring + strategy: + type: RollingUpdate + template: + metadata: + labels: + project: piggymetrics + tier: infrastructure + app: monitoring + spec: + initContainers: + - name: init-skywalking-agent + image: koderover.tencentcloudcr.com/koderover-demo/skywalking-agent:8.8.0 + command: + - 'sh' + - '-c' + - 'set -ex;mkdir -p /vmskywalking/agent;cp -r /usr/skywalking/agent/* /vmskywalking/agent;' + volumeMounts: + - mountPath: /vmskywalking/agent + name: skywalking-agent + containers: + - env: + - name: JAVA_OPTS + value: "-javaagent:/opt/skywalking/agent/skywalking-agent.jar -Duser.timezone=GMT+8" + - name: SW_AGENT_NAME + value: "monitoring" + - name: SW_AGENT_COLLECTOR_BACKEND_SERVICES + value: "skywalking-oap.skywalking.svc.cluster.local:11800" + - name: CONFIG_SERVICE_PASSWORD + valueFrom: + secretKeyRef: + name: piggymetrics + key: config_service_password + envFrom: + - configMapRef: + name: special-config-env + image: koderover.tencentcloudcr.com/koderover-demo/piggymetrics-monitoring:v0.3.0 + name: monitoring + resources: + limits: + cpu: 500m + memory: 1G + requests: + cpu: 100m + memory: 256M + ports: + - containerPort: 8080 + imagePullPolicy: Always + volumeMounts: + - mountPath: /opt/skywalking/agent + name: skywalking-agent + restartPolicy: Always + volumes: + - name: skywalking-agent + emptyDir: {} diff --git a/examples/spring-cloud-piggymetrics/notification-mongodb-service/11-notification-mongodb-service.yaml b/examples/spring-cloud-piggymetrics/notification-mongodb-service/11-notification-mongodb-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..813d6f62369125e9218655e748e6f836ee6cef1e --- /dev/null +++ b/examples/spring-cloud-piggymetrics/notification-mongodb-service/11-notification-mongodb-service.yaml @@ -0,0 +1,61 @@ +apiVersion: v1 +kind: Service +metadata: + name: notification-mongodb + labels: + project: piggymetrics + tier: database + app: notification-mongodb +spec: + ports: + - name: mongo + port: 27017 + targetPort: 27017 + selector: + project: piggymetrics + tier: database + app: notification-mongodb +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + project: piggymetrics + tier: database + app: notification-mongodb + name: notification-mongodb +spec: + replicas: 1 + selector: + matchLabels: + project: piggymetrics + tier: database + app: notification-mongodb + strategy: + type: Recreate + template: + metadata: + labels: + project: piggymetrics + tier: database + app: notification-mongodb + spec: + containers: + - env: + - name: MONGODB_PASSWORD + valueFrom: + secretKeyRef: + name: piggymetrics + key: notification_mongodb_password + ports: + - containerPort: 27017 + image: koderover.tencentcloudcr.com/koderover-demo/piggymetrics-mongodb:v0.1.1 + name: notification-mongodb + resources: + limits: + cpu: 500m + memory: 1G + requests: + cpu: 100m + memory: 256M + restartPolicy: Always diff --git a/examples/spring-cloud-piggymetrics/notification-service/12-notification-service.yaml b/examples/spring-cloud-piggymetrics/notification-service/12-notification-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e0fae28ba787887cd96bbc10716ec6dde6049c28 --- /dev/null +++ b/examples/spring-cloud-piggymetrics/notification-service/12-notification-service.yaml @@ -0,0 +1,117 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + project: piggymetrics + tier: backend + app: notification-service + name: notification-service +spec: + ports: + - name: http + port: 8000 + targetPort: 8000 + selector: + project: piggymetrics + tier: backend + app: notification-service +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + project: piggymetrics + tier: backend + app: notification-service + name: notification-service +spec: + replicas: 1 + selector: + matchLabels: + project: piggymetrics + tier: backend + app: notification-service + strategy: + type: RollingUpdate + template: + metadata: + labels: + project: piggymetrics + tier: backend + app: notification-service + spec: + initContainers: + - name: init-skywalking-agent + image: koderover.tencentcloudcr.com/koderover-demo/skywalking-agent:8.8.0 + command: + - 'sh' + - '-c' + - 'set -ex;mkdir -p /vmskywalking/agent;cp -r /usr/skywalking/agent/* /vmskywalking/agent;' + volumeMounts: + - mountPath: /vmskywalking/agent + name: skywalking-agent + containers: + - env: + - name: JAVA_OPTS + value: "-javaagent:/opt/skywalking/agent/skywalking-agent.jar -Duser.timezone=GMT+8" + - name: SW_AGENT_NAME + value: "notification-service" + - name: SW_AGENT_COLLECTOR_BACKEND_SERVICES + value: "skywalking-oap.skywalking.svc.cluster.local:11800" + - name: CONFIG_SERVICE_PASSWORD + valueFrom: + secretKeyRef: + name: piggymetrics + key: config_service_password + - name: MONGODB_PASSWORD + valueFrom: + secretKeyRef: + name: piggymetrics + key: notification_mongodb_password + - name: NOTIFICATION_SERVICE_PASSWORD + valueFrom: + secretKeyRef: + name: piggymetrics + key: notification_service_password + - name: NOTIFICATION_EMAIL_HOST + valueFrom: + secretKeyRef: + name: piggymetrics + key: notification_email_host + - name: NOTIFICATION_EMAIL_PORT + valueFrom: + secretKeyRef: + name: piggymetrics + key: notification_email_port + - name: NOTIFICATION_EMAIL_USER + valueFrom: + secretKeyRef: + name: piggymetrics + key: notification_email_user + - name: NOTIFICATION_EMAIL_PASS + valueFrom: + secretKeyRef: + name: piggymetrics + key: notification_email_pass + envFrom: + - configMapRef: + name: special-config-env + ports: + - containerPort: 8000 + image: koderover.tencentcloudcr.com/koderover-demo/piggymetrics-notification-service:v0.3.0 + name: notification-service + resources: + limits: + cpu: "1" + memory: 1G + requests: + cpu: 500m + memory: 512M + imagePullPolicy: Always + volumeMounts: + - mountPath: /opt/skywalking/agent + name: skywalking-agent + restartPolicy: Always + volumes: + - name: skywalking-agent + emptyDir: {} diff --git a/examples/spring-cloud-piggymetrics/rabbitmq-service/1-rabbitmq-service.yaml b/examples/spring-cloud-piggymetrics/rabbitmq-service/1-rabbitmq-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..73c49f32a4740e5aaa3e4d9d6fefbfad354034bf --- /dev/null +++ b/examples/spring-cloud-piggymetrics/rabbitmq-service/1-rabbitmq-service.yaml @@ -0,0 +1,59 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + project: piggymetrics + tier: infrastructure + app: rabbitmq + name: rabbitmq +spec: + ports: + - name: http + port: 15672 + targetPort: 15672 + - name: amqp + port: 5672 + targetPort: 5672 + selector: + project: piggymetrics + tier: infrastructure + app: rabbitmq +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + project: piggymetrics + tier: infrastructure + app: rabbitmq + name: rabbitmq +spec: + replicas: 1 + selector: + matchLabels: + project: piggymetrics + tier: infrastructure + app: rabbitmq + strategy: + type: RollingUpdate + template: + metadata: + labels: + project: piggymetrics + tier: infrastructure + app: rabbitmq + spec: + containers: + - image: rabbitmq:3-management + name: rabbitmq + ports: + - containerPort: 15672 + - containerPort: 5672 + resources: + limits: + cpu: 500m + memory: 1G + requests: + cpu: 100m + memory: 256M + restartPolicy: Always \ No newline at end of file diff --git a/examples/spring-cloud-piggymetrics/registry-service/3-registry-service.yaml b/examples/spring-cloud-piggymetrics/registry-service/3-registry-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1667036bca116c8723540c058434441fec435ac4 --- /dev/null +++ b/examples/spring-cloud-piggymetrics/registry-service/3-registry-service.yaml @@ -0,0 +1,87 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + project: piggymetrics + tier: infrastructure + app: registry + name: registry +spec: + ports: + - name: http + port: 8761 + targetPort: 8761 + selector: + project: piggymetrics + tier: infrastructure + app: registry +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + project: piggymetrics + tier: infrastructure + app: registry + name: registry +spec: + replicas: 1 + selector: + matchLabels: + project: piggymetrics + tier: infrastructure + app: registry + strategy: + type: RollingUpdate + template: + metadata: + labels: + project: piggymetrics + tier: infrastructure + app: registry + spec: + initContainers: + - name: init-skywalking-agent + image: koderover.tencentcloudcr.com/koderover-demo/skywalking-agent:8.8.0 + command: + - 'sh' + - '-c' + - 'set -ex;mkdir -p /vmskywalking/agent;cp -r /usr/skywalking/agent/* /vmskywalking/agent;' + volumeMounts: + - mountPath: /vmskywalking/agent + name: skywalking-agent + containers: + - env: + - name: JAVA_OPTS + value: "-javaagent:/opt/skywalking/agent/skywalking-agent.jar -Duser.timezone=GMT+8" + - name: SW_AGENT_NAME + value: "registry" + - name: SW_AGENT_COLLECTOR_BACKEND_SERVICES + value: "skywalking-oap.skywalking.svc.cluster.local:11800" + - name: CONFIG_SERVICE_PASSWORD + valueFrom: + secretKeyRef: + name: piggymetrics + key: config_service_password + envFrom: + - configMapRef: + name: special-config-env + image: koderover.tencentcloudcr.com/koderover-demo/piggymetrics-registry:v0.3.0 + name: registry + resources: + limits: + cpu: 800m + memory: 1G + requests: + cpu: 400m + memory: 768M + imagePullPolicy: Always + ports: + - containerPort: 8761 + volumeMounts: + - mountPath: /opt/skywalking/agent + name: skywalking-agent + restartPolicy: Always + volumes: + - name: skywalking-agent + emptyDir: {} diff --git a/examples/spring-cloud-piggymetrics/statistics-mongodb-service/9-statistics-mongodb-service.yaml b/examples/spring-cloud-piggymetrics/statistics-mongodb-service/9-statistics-mongodb-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ee10305ad17b5683e4762a12be2f01bc6791bc3b --- /dev/null +++ b/examples/spring-cloud-piggymetrics/statistics-mongodb-service/9-statistics-mongodb-service.yaml @@ -0,0 +1,61 @@ +apiVersion: v1 +kind: Service +metadata: + name: statistics-mongodb + labels: + project: piggymetrics + tier: database + app: statistics-mongodb +spec: + ports: + - name: mongo + port: 27017 + targetPort: 27017 + selector: + project: piggymetrics + tier: database + app: statistics-mongodb +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + project: piggymetrics + tier: database + app: statistics-mongodb + name: statistics-mongodb +spec: + replicas: 1 + selector: + matchLabels: + project: piggymetrics + tier: database + app: statistics-mongodb + strategy: + type: Recreate + template: + metadata: + labels: + project: piggymetrics + tier: database + app: statistics-mongodb + spec: + containers: + - env: + - name: MONGODB_PASSWORD + valueFrom: + secretKeyRef: + name: piggymetrics + key: statistics_mongodb_password + ports: + - containerPort: 27017 + image: koderover.tencentcloudcr.com/koderover-demo/piggymetrics-mongodb:v0.1.1 + name: statistics-mongodb + resources: + limits: + cpu: 500m + memory: 1G + requests: + cpu: 100m + memory: 256M + restartPolicy: Always diff --git a/examples/spring-cloud-piggymetrics/statistics-services/10-statistics-service.yaml b/examples/spring-cloud-piggymetrics/statistics-services/10-statistics-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a830c2ce5407a2d96989c0f44ba23d9682c274f --- /dev/null +++ b/examples/spring-cloud-piggymetrics/statistics-services/10-statistics-service.yaml @@ -0,0 +1,97 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + project: piggymetrics + tier: backend + app: statistics-service + name: statistics-service +spec: + ports: + - name: http + port: 7000 + targetPort: 7000 + selector: + project: piggymetrics + tier: backend + app: statistics-service +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + project: piggymetrics + tier: backend + app: statistics-service + name: statistics-service +spec: + replicas: 1 + selector: + matchLabels: + project: piggymetrics + tier: backend + app: statistics-service + strategy: + type: RollingUpdate + template: + metadata: + labels: + project: piggymetrics + tier: backend + app: statistics-service + spec: + initContainers: + - name: init-skywalking-agent + image: koderover.tencentcloudcr.com/koderover-demo/skywalking-agent:8.8.0 + command: + - 'sh' + - '-c' + - 'set -ex;mkdir -p /vmskywalking/agent;cp -r /usr/skywalking/agent/* /vmskywalking/agent;' + volumeMounts: + - mountPath: /vmskywalking/agent + name: skywalking-agent + containers: + - env: + - name: JAVA_OPTS + value: "-javaagent:/opt/skywalking/agent/skywalking-agent.jar -Duser.timezone=GMT+8" + - name: SW_AGENT_NAME + value: "statistics-service" + - name: SW_AGENT_COLLECTOR_BACKEND_SERVICES + value: "skywalking-oap.skywalking.svc.cluster.local:11800" + - name: CONFIG_SERVICE_PASSWORD + valueFrom: + secretKeyRef: + name: piggymetrics + key: config_service_password + - name: MONGODB_PASSWORD + valueFrom: + secretKeyRef: + name: piggymetrics + key: statistics_mongodb_password + - name: STATISTICS_SERVICE_PASSWORD + valueFrom: + secretKeyRef: + name: piggymetrics + key: statistics_service_password + envFrom: + - configMapRef: + name: special-config-env + ports: + - containerPort: 7000 + image: koderover.tencentcloudcr.com/koderover-demo/piggymetrics-statistics-service:v0.3.0 + name: statistics-service + resources: + limits: + cpu: "1" + memory: 1G + requests: + cpu: 500m + memory: 512M + imagePullPolicy: Always + volumeMounts: + - mountPath: /opt/skywalking/agent + name: skywalking-agent + restartPolicy: Always + volumes: + - name: skywalking-agent + emptyDir: {} diff --git a/examples/spring-cloud-piggymetrics/turbine-stream-service/14-turbine-stream-service.yaml b/examples/spring-cloud-piggymetrics/turbine-stream-service/14-turbine-stream-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a84f38457f170b81d5b7d3edaf8a6930110d5a8 --- /dev/null +++ b/examples/spring-cloud-piggymetrics/turbine-stream-service/14-turbine-stream-service.yaml @@ -0,0 +1,92 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + project: piggymetrics + trier: infrastructure + app: turbine-stream-service + name: turbine-stream-service +spec: + ports: + - name: exposed + port: 8989 + targetPort: 8989 + - name: http + port: 8080 + targetPort: 8080 + selector: + project: piggymetrics + trier: infrastructure + app: turbine-stream-service +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + project: piggymetrics + trier: infrastructure + app: turbine-stream-service + name: turbine-stream-service +spec: + replicas: 1 + selector: + matchLabels: + project: piggymetrics + trier: infrastructure + app: turbine-stream-service + strategy: + type: RollingUpdate + template: + metadata: + labels: + project: piggymetrics + trier: infrastructure + app: turbine-stream-service + spec: + initContainers: + - name: init-skywalking-agent + image: koderover.tencentcloudcr.com/koderover-demo/skywalking-agent:8.8.0 + command: + - 'sh' + - '-c' + - 'set -ex;mkdir -p /vmskywalking/agent;cp -r /usr/skywalking/agent/* /vmskywalking/agent;' + volumeMounts: + - mountPath: /vmskywalking/agent + name: skywalking-agent + containers: + - env: + - name: JAVA_OPTS + value: "-javaagent:/opt/skywalking/agent/skywalking-agent.jar -Duser.timezone=GMT+8" + - name: SW_AGENT_NAME + value: "turbine-stream-service" + - name: SW_AGENT_COLLECTOR_BACKEND_SERVICES + value: "skywalking-oap.skywalking.svc.cluster.local:11800" + - name: CONFIG_SERVICE_PASSWORD + valueFrom: + secretKeyRef: + name: piggymetrics + key: config_service_password + envFrom: + - configMapRef: + name: special-config-env + image: koderover.tencentcloudcr.com/koderover-demo/piggymetrics-statistics-service:v0.3.0 + name: turbine-stream-service + resources: + limits: + cpu: "1" + memory: 1G + requests: + cpu: 500m + memory: 512M + ports: + - containerPort: 8989 + - containerPort: 8080 + imagePullPolicy: Always + volumeMounts: + - mountPath: /opt/skywalking/agent + name: skywalking-agent + restartPolicy: Always + volumes: + - name: skywalking-agent + emptyDir: {} + diff --git a/examples/test-demo/go.mod b/examples/test-demo/go.mod new file mode 100644 index 0000000000000000000000000000000000000000..dfd6ffbdfaef25b40587dd26f921a278bef3c858 --- /dev/null +++ b/examples/test-demo/go.mod @@ -0,0 +1,17 @@ +module test-demo + +go 1.18 + +require ( + github.com/onsi/ginkgo/v2 v2.5.0 + github.com/onsi/gomega v1.24.0 +) + +require ( + github.com/go-logr/logr v1.2.3 // indirect + github.com/google/go-cmp v0.5.9 // indirect + golang.org/x/net v0.1.0 // indirect + golang.org/x/sys v0.1.0 // indirect + golang.org/x/text v0.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/examples/test-demo/go.sum b/examples/test-demo/go.sum new file mode 100644 index 0000000000000000000000000000000000000000..9cf15e16468ab93457003651eef728b9a0f2767e --- /dev/null +++ b/examples/test-demo/go.sum @@ -0,0 +1,20 @@ +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/onsi/ginkgo/v2 v2.5.0 h1:TRtrvv2vdQqzkwrQ1ke6vtXf7IK34RBUJafIy1wMwls= +github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= +github.com/onsi/gomega v1.24.0 h1:+0glovB9Jd6z3VR+ScSwQqXVTIfJcGA9UBM8yzQxhqg= +github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= +golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/examples/test-demo/test-multi-envs/multi_env_test.go b/examples/test-demo/test-multi-envs/multi_env_test.go new file mode 100644 index 0000000000000000000000000000000000000000..d315662106544b1020f141d765b62ca1038ce927 --- /dev/null +++ b/examples/test-demo/test-multi-envs/multi_env_test.go @@ -0,0 +1,86 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test_multi_envs + +import ( + "net/http" + "os" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestMultiEnvs(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "TestDemo Suite") +} + +type SiteConfig struct { + WWWSite string + DocSite string + InvalidSite string +} + +func GetConfig() SiteConfig { + if os.Getenv("ENV_NAME") == "test" { + return SiteConfig{ + WWWSite: "https://www.koderover-test.com", + DocSite: "https://docs.koderover-test.com", + InvalidSite: "https://www.koderover-test.com.cn", + } + } + return SiteConfig{ + WWWSite: "https://www.koderover.com", + DocSite: "https://docs.koderover.com", + InvalidSite: "https://www.koderover.com.cn", + } +} + +func LifeProbe(host string) (int, error) { + req, _ := http.NewRequest("GET", host, nil) + req.Header.Set("Content-Type", "application/json") + + client := &http.Client{} + response, err := client.Do(req) + if err != nil { + return 0, err + } + return response.StatusCode, err +} + +var _ = Describe("ZadigWebsitesLifeProbe", func() { + It("Case1: WWW website should be OK", func() { + host := GetConfig().WWWSite + code, err := LifeProbe(host) + Expect(err).Should(BeNil()) + Expect(code).Should(Equal(200)) + }) + + It("Case2: Docs website should be OK", func() { + host := GetConfig().DocSite + code, err := LifeProbe(host) + Expect(err).Should(BeNil()) + Expect(code).Should(Equal(200)) + }) + + It("Case3: Sorry, invalid site won't work", func() { + host := GetConfig().InvalidSite + _, err := LifeProbe(host) + Expect(err).Should(Not(BeNil())) + }) +}) diff --git a/examples/test-demo/test/demo_test.go b/examples/test-demo/test/demo_test.go new file mode 100644 index 0000000000000000000000000000000000000000..2b80996c90b23f267ac5398ba9f49944abf8cd5d --- /dev/null +++ b/examples/test-demo/test/demo_test.go @@ -0,0 +1,64 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "net/http" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestDemo(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "TestDemo Suite") +} + +func HelloGeek(host string) (int, error) { + req, _ := http.NewRequest("GET", host, nil) + req.Header.Set("Content-Type", "application/json") + + client := &http.Client{} + response, err := client.Do(req) + if err != nil { + return 0, err + } + return response.StatusCode, err +} + +var _ = Describe("ZadigWebsitesLifeProbe", func() { + It("Case1: Hello, Welcome to https://www.koderover.com", func() { + host := "https://www.koderover.com" + code, err := HelloGeek(host) + Expect(err).Should(BeNil()) + Expect(code).Should(Equal(200)) + }) + + It("Case2: Hello, Welcome to https://docs.koderover.com/", func() { + host := "https://docs.koderover.com/" + code, err := HelloGeek(host) + Expect(err).Should(BeNil()) + Expect(code).Should(Equal(200)) + }) + + It("Case3: Sorry, https://www.koderover.com.cn won't work", func() { + host := "https://www.koderover.com.cn" + _, err := HelloGeek(host) + Expect(err).Should(Not(BeNil())) + }) +}) diff --git a/examples/voting-app/chart/Chart.yaml b/examples/voting-app/chart/Chart.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fcc9eb19e1e873dde56eab727895a3571b6de5d5 --- /dev/null +++ b/examples/voting-app/chart/Chart.yaml @@ -0,0 +1,23 @@ +apiVersion: v2 +name: voting-app +description: multi microservice project + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 1.0.7 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +appVersion: 1.16.0 diff --git a/examples/voting-app/chart/templates/db.yaml b/examples/voting-app/chart/templates/db.yaml new file mode 100644 index 0000000000000000000000000000000000000000..58baacd074dd798d0126aa1f45fb74291b645ad9 --- /dev/null +++ b/examples/voting-app/chart/templates/db.yaml @@ -0,0 +1,53 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: db + name: db +spec: + replicas: {{ .Values.services.db.replicas }} + selector: + matchLabels: + app: db + template: + metadata: + labels: + app: db + spec: + containers: + - image: {{ .Values.services.db.image }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + name: postgres + ports: + - containerPort: {{ .Values.services.db.port }} + name: postgres + volumeMounts: + - mountPath: /var/lib/postgresql/data + name: db-data + env: + - name: POSTGRES_HOST_AUTH_METHOD + value: trust + resources: + limits: + memory: {{ .Values.resources.limits.memory }} + cpu: {{ .Values.resources.limits.cpu }} + volumes: + - name: db-data + emptyDir: {} + imagePullSecrets: + - name: {{ .Values.imagePullSecretsName }} +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: db + name: db +spec: + type: ClusterIP + ports: + - name: "db-service" + port: {{ .Values.services.db.port }} + targetPort: {{ .Values.services.db.port }} + selector: + app: db diff --git a/examples/voting-app/chart/templates/redis.yaml b/examples/voting-app/chart/templates/redis.yaml new file mode 100644 index 0000000000000000000000000000000000000000..667723e6a503bb9a8efd6d09426e31efa854a2fc --- /dev/null +++ b/examples/voting-app/chart/templates/redis.yaml @@ -0,0 +1,50 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: redis + name: redis +spec: + replicas: {{ .Values.services.redis.replicas }} + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + spec: + containers: + - image: {{ .Values.services.redis.image }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + name: redis + ports: + - containerPort: {{ .Values.services.redis.port }} + name: redis + volumeMounts: + - mountPath: /data + name: redis-data + resources: + limits: + memory: {{ .Values.resources.limits.memory }} + cpu: {{ .Values.resources.limits.cpu }} + volumes: + - name: redis-data + emptyDir: {} + imagePullSecrets: + - name: {{ .Values.imagePullSecretsName }} +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: redis + name: redis +spec: + type: ClusterIP + ports: + - name: "redis-service" + port: {{ .Values.services.redis.port }} + targetPort: {{ .Values.services.redis.port }} + selector: + app: redis diff --git a/examples/voting-app/chart/templates/result.yaml b/examples/voting-app/chart/templates/result.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e514cfcb9543d4c0e470ff2f8a7cc12411bd7a07 --- /dev/null +++ b/examples/voting-app/chart/templates/result.yaml @@ -0,0 +1,46 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: result + name: result +spec: + replicas: {{ .Values.services.vote.replicas }} + selector: + matchLabels: + app: result + template: + metadata: + labels: + app: result + spec: + containers: + - image: {{ .Values.services.result.image }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + name: result-e2e + ports: + - containerPort: 80 + name: result + resources: + limits: + memory: {{ .Values.resources.limits.memory }} + cpu: {{ .Values.resources.limits.cpu }} + imagePullSecrets: + - name: {{ .Values.imagePullSecretsName }} +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: result + version: rc-origin + name: result +spec: + type: NodePort + ports: + - name: "result-service" + port: {{ .Values.services.result.port }} + targetPort: 80 + selector: + app: result + diff --git a/examples/voting-app/chart/templates/vote.yaml b/examples/voting-app/chart/templates/vote.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8bb6a492b884706d230ab975ef068dcf8376e3b3 --- /dev/null +++ b/examples/voting-app/chart/templates/vote.yaml @@ -0,0 +1,48 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: vote + name: vote +spec: + replicas: {{ .Values.services.vote.replicas }} + selector: + matchLabels: + app: vote + version: rc-origin + template: + metadata: + labels: + app: vote + version: rc-origin + spec: + containers: + - image: {{ .Values.services.vote.image }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + name: vote-e2e + ports: + - containerPort: 80 + name: vote + resources: + limits: + memory: {{ .Values.resources.limits.memory }} + cpu: {{ .Values.resources.limits.cpu }} + imagePullSecrets: + - name: {{ .Values.imagePullSecretsName }} +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: vote + name: vote +spec: + type: NodePort + ports: + - name: "vote-service" + port: {{ .Values.services.vote.port }} + targetPort: 80 + selector: + app: vote + version: rc-origin + diff --git a/examples/voting-app/chart/templates/worker.yaml b/examples/voting-app/chart/templates/worker.yaml new file mode 100644 index 0000000000000000000000000000000000000000..83c89e4bee6051f005e2f9ee41282f09b1ee4628 --- /dev/null +++ b/examples/voting-app/chart/templates/worker.yaml @@ -0,0 +1,26 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: worker + name: worker +spec: + replicas: {{ .Values.services.worker.replicas }} + selector: + matchLabels: + app: worker + template: + metadata: + labels: + app: worker + spec: + containers: + - image: {{ .Values.services.worker.image }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + name: worker-e2e + resources: + limits: + memory: {{ .Values.resources.limits.memory }} + cpu: {{ .Values.resources.limits.cpu }} + imagePullSecrets: + - name: {{ .Values.imagePullSecretsName }} diff --git a/examples/voting-app/chart/values.yaml b/examples/voting-app/chart/values.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50b49eccec76559e05a33be87a609321e530e57a --- /dev/null +++ b/examples/voting-app/chart/values.yaml @@ -0,0 +1,36 @@ +# Default values for chart. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +imagePullPolicy: Always +imagePullSecretsName: default-registry-secret + +services: + vote: + port: 5000 + image: dockersamples/examplevotingapp_vote:before + replicas: 1 + + result: + port: 5001 + image: dockersamples/examplevotingapp_result:before + replicas: 1 + + worker: + image: dockersamples/examplevotingapp_worker + replicas: 1 + + redis: + port: 6379 + image: redis:alpine + replicas: 1 + + db: + port: 5432 + image: postgres:9.4 + replicas: 1 + +resources: + limits: + cpu: 50m + memory: 50Mi \ No newline at end of file diff --git a/examples/voting-app/freestyle-k8s-specifications/db/db-deployment.yaml b/examples/voting-app/freestyle-k8s-specifications/db/db-deployment.yaml index c832029bae6322e86a6685af6b67633d5cf6d1f5..6cd12f16fb7dd45f7e5e5fde989a16198dad315a 100644 --- a/examples/voting-app/freestyle-k8s-specifications/db/db-deployment.yaml +++ b/examples/voting-app/freestyle-k8s-specifications/db/db-deployment.yaml @@ -1,4 +1,4 @@ -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: labels: @@ -26,6 +26,11 @@ spec: env: - name: POSTGRES_HOST_AUTH_METHOD value: trust + resources: + limits: + memory: 50Mi + cpu: 50m volumes: - name: db-data emptyDir: {} + diff --git a/examples/voting-app/freestyle-k8s-specifications/redis/redis-deployment.yaml b/examples/voting-app/freestyle-k8s-specifications/redis/redis-deployment.yaml index 24aa52135fd805412bfdf04182bf1fe7b14623b5..e341929d816aca63367a512d59538ddec53b8d22 100644 --- a/examples/voting-app/freestyle-k8s-specifications/redis/redis-deployment.yaml +++ b/examples/voting-app/freestyle-k8s-specifications/redis/redis-deployment.yaml @@ -23,6 +23,11 @@ spec: volumeMounts: - mountPath: /data name: redis-data + resources: + limits: + memory: 50Mi + cpu: 50m volumes: - name: redis-data emptyDir: {} + diff --git a/examples/voting-app/freestyle-k8s-specifications/result/result-deployment.yaml b/examples/voting-app/freestyle-k8s-specifications/result/result-deployment.yaml index 585b801989bba21c261592fb1575651bccc8ed1b..b6d224cd9eeb2691c683772d7cb7ded292200018 100644 --- a/examples/voting-app/freestyle-k8s-specifications/result/result-deployment.yaml +++ b/examples/voting-app/freestyle-k8s-specifications/result/result-deployment.yaml @@ -20,3 +20,7 @@ spec: ports: - containerPort: 80 name: result + resources: + limits: + memory: 60Mi + cpu: 60m diff --git a/examples/voting-app/freestyle-k8s-specifications/result/result-service.yaml b/examples/voting-app/freestyle-k8s-specifications/result/result-service.yaml index b7af093fb1494891c96f4822b7b592b8de6f8538..4d3cccc06311e55e599ef149ded91ea28ff05d10 100644 --- a/examples/voting-app/freestyle-k8s-specifications/result/result-service.yaml +++ b/examples/voting-app/freestyle-k8s-specifications/result/result-service.yaml @@ -14,7 +14,7 @@ spec: selector: app: result --- -apiVersion: extensions/v1beta1 +apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: result @@ -26,6 +26,9 @@ spec: http: paths: - path: / + pathType: Prefix backend: - serviceName: result - servicePort: 5001 \ No newline at end of file + service: + name: result + port: + number: 5001 diff --git a/examples/voting-app/freestyle-k8s-specifications/vote/vote-deployment.yaml b/examples/voting-app/freestyle-k8s-specifications/vote/vote-deployment.yaml index 4b28134b437c3ac85f263d57771b9113ce4a993b..5f7ca3b12fa61724465b31a9f8151a9f265974a6 100644 --- a/examples/voting-app/freestyle-k8s-specifications/vote/vote-deployment.yaml +++ b/examples/voting-app/freestyle-k8s-specifications/vote/vote-deployment.yaml @@ -22,3 +22,7 @@ spec: ports: - containerPort: 80 name: vote + resources: + limits: + memory: 50Mi + cpu: 50m diff --git a/examples/voting-app/freestyle-k8s-specifications/vote/vote-service.yaml b/examples/voting-app/freestyle-k8s-specifications/vote/vote-service.yaml index 78643aa62985a30901348377906ae3b9b04d5494..a598a818b9717dcb0e321ad1d946023380f83ee2 100644 --- a/examples/voting-app/freestyle-k8s-specifications/vote/vote-service.yaml +++ b/examples/voting-app/freestyle-k8s-specifications/vote/vote-service.yaml @@ -14,7 +14,7 @@ spec: app: vote version: rc-origin --- -apiVersion: extensions/v1beta1 +apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: vote @@ -26,6 +26,9 @@ spec: http: paths: - path: / + pathType: Prefix backend: - serviceName: vote - servicePort: 5000 \ No newline at end of file + service: + name: vote + port: + number: 5000 diff --git a/examples/voting-app/freestyle-k8s-specifications/worker/worker-deployment.yaml b/examples/voting-app/freestyle-k8s-specifications/worker/worker-deployment.yaml index b48cd0b618bc006563f13c69654969df6d302420..c2ac3d86b300c5b5f4f9e4c643767df750ba8f87 100644 --- a/examples/voting-app/freestyle-k8s-specifications/worker/worker-deployment.yaml +++ b/examples/voting-app/freestyle-k8s-specifications/worker/worker-deployment.yaml @@ -17,3 +17,7 @@ spec: containers: - image: dockersamples/examplevotingapp_worker name: worker-e2e + resources: + limits: + memory: 50Mi + cpu: 50m diff --git a/examples/voting-app/function-test/fail/result_1.xml b/examples/voting-app/function-test/fail/result_1.xml index abe52156abe0e59727e2196985412f8c7a135f9a..b56b4455f6074b6f119cb2177579e341d070b275 100644 --- a/examples/voting-app/function-test/fail/result_1.xml +++ b/examples/voting-app/function-test/fail/result_1.xml @@ -4,11 +4,11 @@ - + - - + + - - + + \ No newline at end of file diff --git a/examples/voting-app/function-test/pass/result_1.xml b/examples/voting-app/function-test/pass/result_1.xml index abe52156abe0e59727e2196985412f8c7a135f9a..bbe04641bc325f2e80107edd920a65d048b61b0b 100644 --- a/examples/voting-app/function-test/pass/result_1.xml +++ b/examples/voting-app/function-test/pass/result_1.xml @@ -4,11 +4,11 @@ - + - - + + - - + + \ No newline at end of file diff --git a/examples/webhook/trigger_example.yaml b/examples/webhook/trigger_example.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0d4ad2fa4a96541ac1198066e23d2615a3c842e --- /dev/null +++ b/examples/webhook/trigger_example.yaml @@ -0,0 +1,58 @@ +# 定义触发的步骤 +stages: + - build + - deploy + - test +# 构建参数 +build: + - name: service_name # 服务名 + service_module: service_module_name # 服务组件名 + variables: # zadig 系统中需要先声明出来才能指定，否则这边配置也无效 + - name: k1 + value: v1 +# 部署策略 （该阶段不可缺） +deploy: + strategy: single # single(更新指定环境)/dynamic(动态选择空闲环境)/base(更新基准环境，需配置 env_recycle_policy 和 base_env) + envs_name: + - dev + + # strategy 值为 base 时才需配置以下变量;该策略仅pull_request时生效 + # env_recycle_policy: success # success(成功后销毁)/always(每次销毁)/never(每次保留) + # base_env: dev +# 测试参数 +test: + - name: test_name + repo: + strategy: currentRepo # default(在 zadig 平台默认配置的代码仓库信息)/currentRepo(使用当前变动的代码信息) + variables: + - name: k1 + value: v1 +# 触发条件规则 +rules: + branchs: + - 'feature*' + events: + - pull_request + - push + strategy: + # 自动取消，如果您希望只构建最新的提交，则使用这个选项会自动取消队列中的任务 + auto_cancel: true + # 指定服务与代码目录的关系 + match_folders: + match_folders_tree: + - name: service1 + service_module: service1 + file_tree: + - src/service1 + - ci + - name: service2 + service_module: service2 + file_tree: + - src/service2 + - ci +# 缓存配置 +cache_set: + # 不使用 docker 缓存 + ignore_cache: false + # 不使用工作空间缓存 + reset_cache: false \ No newline at end of file diff --git a/examples/website/yaml/website-example.yaml b/examples/website/yaml/website-example.yaml index 86f93ec5eb7b2dad4ffd50e49b6007c118e67e6a..7e343caac73561b4119bffb03c4496597a09fa96 100644 --- a/examples/website/yaml/website-example.yaml +++ b/examples/website/yaml/website-example.yaml @@ -28,6 +28,6 @@ spec: spec: containers: - name: website-example - image: ccr.ccs.tencentyun.com/koderover-public/website-example:1.0.0 + image: koderover.tencentcloudcr.com/koderover-demo/website-example:1.0.0 ports: - containerPort: 80 diff --git a/go.mod b/go.mod index 28512627c35a59b1bda9d3cf7c83c07bec902601..994f576cc920e6a6fa344f7a917d2fd2f8c389c4 100644 --- a/go.mod +++ b/go.mod @@ -3,94 +3,118 @@ module github.com/koderover/zadig go 1.16 require ( + gitee.com/openeuler/go-gitee v0.0.0-20220323055843-285220630cbc github.com/27149chen/afero v1.6.2 github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/RyanCarrier/dijkstra v1.1.0 github.com/andygrunwald/go-gerrit v0.0.0-20171029143327-95b11af228a1 + github.com/antihax/optional v1.0.0 github.com/aws/aws-sdk-go v1.34.28 github.com/blang/semver/v4 v4.0.0 github.com/bndr/gojenkins v1.1.0 github.com/bradleyfalzon/ghinstallation v1.1.1 - github.com/bshuster-repo/logrus-logstash-hook v1.0.0 // indirect github.com/bugsnag/bugsnag-go v2.1.0+incompatible // indirect github.com/bugsnag/panicwrap v1.3.1 // indirect - github.com/cenkalti/backoff/v3 v3.0.0 + github.com/buraksezer/consistent v0.9.0 + github.com/cenkalti/backoff/v4 v4.1.1 + github.com/cespare/xxhash v1.1.0 + github.com/chartmuseum/helm-push v0.10.1 + github.com/containers/image v3.0.2+incompatible github.com/coocood/freecache v1.1.0 + github.com/coreos/go-oidc/v3 v3.0.0 + github.com/dexidp/dex v0.0.0-20210802203454-3fac2ab6bc3b + github.com/dgraph-io/badger/v3 v3.2103.2 github.com/docker/distribution v2.7.1+incompatible - github.com/docker/docker v17.12.0-ce-rc1.0.20200618181300-9dc6525e6118+incompatible + github.com/docker/docker v20.10.11+incompatible github.com/docker/go-connections v0.4.0 github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect github.com/dsnet/compress v0.0.1 // indirect github.com/garyburd/redigo v1.6.2 // indirect - github.com/gin-contrib/sessions v0.0.3 github.com/gin-contrib/sse v0.1.0 github.com/gin-gonic/gin v1.7.2 + github.com/go-co-op/gocron v1.13.0 + github.com/go-ldap/ldap/v3 v3.3.0 github.com/go-resty/resty/v2 v2.6.0 + github.com/go-sql-driver/mysql v1.6.0 github.com/gofrs/uuid v4.0.0+incompatible // indirect + github.com/gogo/protobuf v1.3.2 + github.com/golang-jwt/jwt v3.2.2+incompatible github.com/google/go-github/v35 v35.3.0 github.com/google/uuid v1.2.0 - github.com/gorilla/handlers v1.5.1 // indirect - github.com/gorilla/mux v1.7.3 + github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.4.2 - github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f github.com/hashicorp/go-multierror v1.1.1 github.com/huaweicloud/huaweicloud-sdk-go-v3 v0.0.50 github.com/jasonlvhit/gocron v0.0.0-20171226191223-3c914c8681c3 + github.com/jinzhu/copier v0.3.5 github.com/jinzhu/now v1.1.2 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect - github.com/mittwald/go-helm-client v0.8.0 + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mittwald/go-helm-client v0.8.4 + github.com/moby/buildkit v0.9.1 github.com/nsqio/go-nsq v1.0.7 github.com/nwaples/rardecode v1.0.0 // indirect github.com/onsi/ginkgo v1.16.4 github.com/onsi/gomega v1.15.0 github.com/opencontainers/go-digest v1.0.0 - github.com/opencontainers/selinux v1.6.0 // indirect github.com/otiai10/copy v1.6.0 github.com/pkg/errors v0.9.1 github.com/rfyiamcool/cronlib v1.0.0 github.com/satori/go.uuid v1.2.0 + github.com/shirou/gopsutil/v3 v3.22.5 github.com/sirupsen/logrus v1.8.1 - github.com/spf13/cobra v1.1.3 - github.com/spf13/viper v1.7.0 + github.com/spf13/cobra v1.2.1 + github.com/spf13/pflag v1.0.5 + github.com/spf13/viper v1.8.1 github.com/stevvooe/resumable v0.0.0-20180830230917-22b14a53ba50 // indirect - github.com/stretchr/testify v1.7.0 + github.com/stretchr/testify v1.7.1 github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14 github.com/swaggo/gin-swagger v1.3.0 github.com/swaggo/swag v1.5.1 github.com/xanzy/go-gitlab v0.50.0 - github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/yvasiyarov/go-metrics v0.0.0-20150112132944-c25f46c4b940 // indirect github.com/yvasiyarov/gorelic v0.0.7 // indirect github.com/yvasiyarov/newrelic_platform_go v0.0.0-20160601141957-9c099fbc30e9 // indirect go.mongodb.org/mongo-driver v1.5.0 go.uber.org/zap v1.19.0 - golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 - golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d + golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 + golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 + golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/tools v0.1.5 // indirect + gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect + gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df gopkg.in/mholt/archiver.v3 v3.1.1 gopkg.in/natefinch/lumberjack.v2 v2.0.0 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b - helm.sh/helm/v3 v3.6.2 - k8s.io/api v0.22.1 - k8s.io/apimachinery v0.22.1 - k8s.io/client-go v0.22.1 - k8s.io/kubectl v0.21.0 - k8s.io/utils v0.0.0-20210802155522-efc7438f0176 - sigs.k8s.io/controller-runtime v0.10.0 - sigs.k8s.io/yaml v1.2.0 + gorm.io/driver/mysql v1.1.2 + gorm.io/gorm v1.21.16 + helm.sh/helm/v3 v3.7.2 + istio.io/api v0.0.0-20220304035241-8c47cbbea144 + istio.io/client-go v0.0.0-20220304040955-30b642d5ba34 + k8s.io/api v0.22.4 + k8s.io/apiextensions-apiserver v0.22.4 + k8s.io/apimachinery v0.22.4 + k8s.io/client-go v0.22.4 + k8s.io/kubectl v0.22.4 + k8s.io/kubernetes v1.13.0 + k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a + sigs.k8s.io/controller-runtime v0.10.1 + sigs.k8s.io/yaml v1.3.0 ) replace ( github.com/docker/distribution => github.com/docker/distribution v2.6.0-rc.1.0.20170726174610-edc3ab29cdff+incompatible github.com/docker/docker => github.com/docker/docker v0.0.0-20180502112750-51a9119f6b81 github.com/docker/go-connections => github.com/docker/go-connections v0.3.1-0.20180212134524-7beb39f0b969 + github.com/go-logr/logr => github.com/go-logr/logr v0.4.0 k8s.io/api => k8s.io/api v0.21.0 k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.21.0 k8s.io/apimachinery => k8s.io/apimachinery v0.21.0 k8s.io/cli-runtime => k8s.io/cli-runtime v0.21.0 k8s.io/client-go => k8s.io/client-go v0.21.0 + k8s.io/klog/v2 => k8s.io/klog/v2 v2.9.0 k8s.io/kubectl => k8s.io/kubectl v0.21.0 + oras.land/oras-go => oras.land/oras-go v0.4.0 ) diff --git a/go.sum b/go.sum index 08ed6a88b553ed854ed384718a6612c9524030fd..5a9dc0a090e0cdff6422bffcd2a5a06c068b5b21 100644 --- a/go.sum +++ b/go.sum @@ -1,111 +1,241 @@ bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= +bazil.org/fuse v0.0.0-20180421153158-65cc252bf669/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= +cloud.google.com/go v0.25.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.37.2/go.mod h1:H8IAquKe2L30IxoupDgqTaQvKSwF/c8prYHynGIWQbA= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.39.0/go.mod h1:rVLT6fkc8chs9sfPtFc1SBH6em7n+ZoXaG+87tDISts= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.88.0 h1:MZ2cf9Elnv1wqccq8ooKO2MqHQLc+ChCp/+QWObCpxg= +cloud.google.com/go v0.88.0/go.mod h1:dnKwfYbP9hQhefiUvpbcAyoGSHUrOxR20JVElLiUvEY= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +code.gitea.io/sdk/gitea v0.12.0/go.mod h1:z3uwDV/b9Ls47NGukYM9XhnHtqPh/J+t40lsUrR6JDY= +contrib.go.opencensus.io/exporter/aws v0.0.0-20181029163544-2befc13012d0/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA= +contrib.go.opencensus.io/exporter/ocagent v0.5.0/go.mod h1:ImxhfLRpxoYiSq891pBrLVhN+qmP8BTVvdH2YLs7Gl0= +contrib.go.opencensus.io/exporter/stackdriver v0.12.1/go.mod h1:iwB6wGarfphGGe/e5CWqyUk/cLzKnWsOKPVW3no6OTw= +contrib.go.opencensus.io/integrations/ocsql v0.1.4/go.mod h1:8DsSdjz3F+APR+0z0WkU1aRorQCFfRxvqjUUPMbF3fE= +contrib.go.opencensus.io/resource v0.1.1/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcigGlFvXwEGEnkRLA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +entgo.io/ent v0.8.0/go.mod h1:KNjsukat/NJi6zJh1utwRadsbGOZsBbAZNDxkW7tMCc= +git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +git.apache.org/thrift.git v0.12.0/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +gitee.com/openeuler/go-gitee v0.0.0-20220323055843-285220630cbc h1:xjPHn/Qey+IMUZYo8goKsWmXHjn04RbaLCVinOh4UlM= +gitee.com/openeuler/go-gitee v0.0.0-20220323055843-285220630cbc/go.mod h1:qGJhn1KxC5UE4BUmxCE/hTpFfuKbd3U3V9fNROrspfE= github.com/27149chen/afero v1.6.2 h1:WRPUOreB90qGRQs8fxXcvi0srBPHy2octiJme7C7XCA= github.com/27149chen/afero v1.6.2/go.mod h1:68wFWsZwQPvguVWn5di+KCOLI0KQtSRve2mPFVg6VWY= +github.com/AkihiroSuda/containerd-fuse-overlayfs v1.0.0/go.mod h1:0mMDvQFeLbbn1Wy8P2j3hwFhqBq+FKn8OZPno8WLmp8= +github.com/AppsFlyer/go-sundheit v0.4.0/go.mod h1:iZ8zWMS7idcvmqewf5mEymWWgoOiG/0WD4+aeh+heX4= +github.com/Azure/azure-amqp-common-go/v2 v2.1.0/go.mod h1:R8rea+gJRuJR6QxTir/XuEd+YuKoUiazDC/N96FiDEU= +github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= +github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= +github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v19.1.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v29.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v30.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v35.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v38.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v42.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-service-bus-go v0.9.1/go.mod h1:yzBx6/BUGfjfeqbRZny9AQIbIe3AcV9WZbAdpkoXOa0= +github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest v10.15.5+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest v12.0.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest v14.1.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest v0.9.3/go.mod h1:GsRuLYvwzLjjjRoWEIyMUaYq8GNUx2nRB378IPt/1p0= +github.com/Azure/go-autorest/autorest v0.10.2/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= +github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= +github.com/Azure/go-autorest/autorest/adal v0.8.1/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= +github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= +github.com/Azure/go-autorest/autorest/adal v0.8.3/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/azure/auth v0.4.2/go.mod h1:90gmfKdlmKgfjUpnCEpOJzsUEjrWDSLwHIG73tSXddM= +github.com/Azure/go-autorest/autorest/azure/cli v0.3.1/go.mod h1:ZG5p860J94/0kI9mNJVoIoLgXcirM2gF5i2kWloofxw= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/to v0.2.0/go.mod h1:GunWKJp1AEqgMaGLV+iocmRAJWqST1wQYhyyjXJ3SJc= +github.com/Azure/go-autorest/autorest/to v0.3.0/go.mod h1:MgwOyqaIuKdG4TL/2ywSsIWKAfJfgHDo8ObuUk3t5sA= +github.com/Azure/go-autorest/autorest/validation v0.1.0/go.mod h1:Ha3z/SqBeaalWQvokg3NZAlQTalVMtOIAs1aGK7G6u8= +github.com/Azure/go-autorest/autorest/validation v0.2.0/go.mod h1:3EEqHnBxQGHXRYq3HT1WyXAvT7LLY3tl70hw6tQIbjI= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c h1:/IBSNwUN8+eKzUzbJPqhK839ygXJ82sde8x3ogr6R28= +github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/Djarvur/go-err113 v0.0.0-20200410182137-af658d038157/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs= +github.com/Djarvur/go-err113 v0.1.0/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs= +github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20191009163259-e802c2cb94ae/go.mod h1:mjwGPas4yKduTyubHvD1Atl9r1rUq8DfVy+gkVvZ+oo= +github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20190822182118-27a4ced34534/go.mod h1:iroGtC8B3tQiqtds1l+mgk/BBOrxbqjH+eUfFQYRc14= github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd h1:sjQovDkwrZp8u+gxLtPgKGjk5hCxuy2hrRejBTA9xFU= github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= +github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/semver/v3 v3.0.3/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/Masterminds/semver/v3 v3.1.0/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZCSYp4Z0m2dk6cEM60= +github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8= github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk= -github.com/Masterminds/squirrel v1.5.0 h1:JukIZisrUXadA9pl3rMkjhiamxiB0cXiu+HGp/Y8cY8= github.com/Masterminds/squirrel v1.5.0/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= +github.com/Masterminds/squirrel v1.5.2 h1:UiOEi2ZX4RCSkpiNDQN5kro/XIBpSRk9iTqdIRPzUXE= +github.com/Masterminds/squirrel v1.5.2/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= github.com/Masterminds/vcs v1.13.1/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= +github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= +github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= +github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= +github.com/Microsoft/go-winio v0.4.15-0.20200908182639-5b44b70ab3ab/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= +github.com/Microsoft/go-winio v0.4.15/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= -github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk= github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= -github.com/Microsoft/hcsshim v0.8.14 h1:lbPVK25c1cu5xTLITwpUcxoA9vKrKErASPYygvouJns= +github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.4.17 h1:iT12IBVClFevaf8PuVyi3UmZOVh4OqnaLxDTW2O6j3w= +github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= +github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= +github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= +github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= +github.com/Microsoft/hcsshim v0.8.10/go.mod h1:g5uw8EV2mAlzqe94tfNBNdr89fnbD/n3HV0OhsddkmM= github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= +github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= +github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= +github.com/Microsoft/hcsshim v0.8.18/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= +github.com/Microsoft/hcsshim v0.8.21 h1:btRfUDThBE5IKcvI8O8jOiIkujUsAMBSRsYDYmEi6oM= +github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= +github.com/Microsoft/hcsshim/test v0.0.0-20200826032352-301c83a30e7c/go.mod h1:30A5igQ91GEmhYJF8TaRP79pMBOYynRsyOByfVV0dU4= +github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= +github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM= +github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/RyanCarrier/dijkstra v1.1.0 h1:/NDihjfJA3CxFaZz8EdzTwdFKFZDvvB881OVLdakRcI= github.com/RyanCarrier/dijkstra v1.1.0/go.mod h1:5agGUBNEtUAGIANmbw09fuO3a2htPEkc1jNH01qxCWA= github.com/RyanCarrier/dijkstra-1 v0.0.0-20170512020943-0e5801a26345 h1:fgSpoKViTSqRb4hjDNj10ig5wUvO0CayCzFdLf6fuRM= github.com/RyanCarrier/dijkstra-1 v0.0.0-20170512020943-0e5801a26345/go.mod h1:OK4EvWJ441LQqGzed5NGB6vKBAE34n3z7iayPcEwr30= +github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs= +github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= -github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= +github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/albertorestifo/dijkstra v0.0.0-20160910063646-aba76f725f72 h1:uGeGZl8PxSq8VZGG4QK5njJTFA4/G/x5CYORvQVXtAE= github.com/albertorestifo/dijkstra v0.0.0-20160910063646-aba76f725f72/go.mod h1:o+JdB7VetTHjLhU0N57x18B9voDBQe0paApdEAEoEfw= +github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andygrunwald/go-gerrit v0.0.0-20171029143327-95b11af228a1 h1:2Qp3BQZ7r1z+oMUnFhD6FQ86Hh1PO+59yoUNNJ5Dn0M= github.com/andygrunwald/go-gerrit v0.0.0-20171029143327-95b11af228a1/go.mod h1:0iuRQp6WJ44ts+iihy5E/WlPqfg5RNeQxOmzRkxCdtk= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apex/log v1.1.4/go.mod h1:AlpoD9aScyQfJDVHmLMEcx4oU6LqzkWp4Mg9GdAcEvQ= +github.com/apex/log v1.3.0/go.mod h1:jd8Vpsr46WAe3EZSQ/IUMs2qQD/GOycT5rPWCO1yGcs= +github.com/apex/logs v0.0.4/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= +github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= +github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY= github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= -github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= -github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= +github.com/aws/aws-sdk-go v1.15.90/go.mod h1:es1KtYUFs7le0xQ3rOihkuoVD90z7D0fR2Qm4S00/gU= +github.com/aws/aws-sdk-go v1.16.26/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.19.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.19.45/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.25.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.27.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.31.6/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.34.9/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.34.28 h1:sscPpn/Ns3i0F4HPEWAVcwdIRaZZCuL7llJ2/60yPIk= github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= -github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= +github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= +github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -114,71 +244,197 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= +github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI= +github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bndr/gojenkins v1.1.0 h1:TWyJI6ST1qDAfH33DQb3G4mD8KkrBfyfSUoZBHQAvPI= github.com/bndr/gojenkins v1.1.0/go.mod h1:QeskxN9F/Csz0XV/01IC8y37CapKKWvOHa0UHLLX1fM= -github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff/go.mod h1:+RTT1BOk5P97fT2CiHkbFQwkK3mjsFAP6zCYV2aXtjw= -github.com/bradfitz/gomemcache v0.0.0-20190329173943-551aad21a668/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= +github.com/bombsimon/wsl/v2 v2.0.0/go.mod h1:mf25kr/SqFEPhhcxW1+7pxzGlW+hIl/hYTKY95VwV8U= +github.com/bombsimon/wsl/v2 v2.2.0/go.mod h1:Azh8c3XGEJl9LyX0/sFC+CKMc7Ssgua0g+6abzXN4Pg= +github.com/bombsimon/wsl/v3 v3.0.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc= +github.com/bombsimon/wsl/v3 v3.1.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc= +github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/bradleyfalzon/ghinstallation v1.1.1 h1:pmBXkxgM1WeF8QYvDLT5kuQiHMcmf+X015GI0KM/E3I= github.com/bradleyfalzon/ghinstallation v1.1.1/go.mod h1:vyCmHTciHx/uuyN82Zc3rXN3X2KTK8nUTCrTMwAhcug= -github.com/bradleypeabody/gorilla-sessions-memcache v0.0.0-20181103040241-659414f458e1/go.mod h1:dkChI7Tbtx7H1Tj7TqGSZMOeGpMP5gLHtjroHd4agiI= github.com/bshuster-repo/logrus-logstash-hook v1.0.0 h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70= github.com/bshuster-repo/logrus-logstash-hook v1.0.0/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= +github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= github.com/bugsnag/bugsnag-go v2.1.0+incompatible h1:SuqsBHDutts2rZh4swHEWTexxi0F/JZ/6j1rR9BFe7I= github.com/bugsnag/bugsnag-go v2.1.0+incompatible/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= +github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= +github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= github.com/bugsnag/panicwrap v1.3.1 h1:pmuhHlhbUV4OOrGDvoiMjHSZzwRcL+I9cIzYKiW4lII= github.com/bugsnag/panicwrap v1.3.1/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= -github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= -github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= -github.com/cenkalti/backoff/v3 v3.0.0 h1:ske+9nBpD9qZsTBoF41nW5L+AIuFBKMeze18XQ3eG1c= -github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= +github.com/buraksezer/consistent v0.9.0 h1:Zfs6bX62wbP3QlbPGKUhqDw7SmNkOzY5bHZIYXYpR5g= +github.com/buraksezer/consistent v0.9.0/go.mod h1:6BrVajWq7wbKZlTOUPs/XVfR8c0maujuPowduSpZqmw= +github.com/caarlos0/ctrlc v1.0.0/go.mod h1:CdXpj4rmq0q/1Eb44M9zi2nKB0QraNKuRGYGrrHhcQw= +github.com/campoy/unique v0.0.0-20180121183637-88950e537e7e/go.mod h1:9IOqJGCPMSc6E5ydlp5NIonxObaeu/Iub/X03EKPVYo= +github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e/go.mod h1:oDpT4efm8tSYHXV5tHSdRvBet/b/QzxZ+XyyPehvm3A= +github.com/cenkalti/backoff/v4 v4.1.1 h1:G2HAfAmvm/GcKan2oOQpBXOd2tT2G57ZnZGWa1PxPBQ= +github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= +github.com/chartmuseum/helm-push v0.10.1 h1:NqStAmarEy0GnrDCk0zubGKeRmkhOm/rRi5au8h76BA= +github.com/chartmuseum/helm-push v0.10.1/go.mod h1:s6xTICU31jKdLkOXS+GgaR61E+oU4h8TWb1yZcHq8OE= +github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= +github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= -github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= +github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc= +github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= +github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= +github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59 h1:qWj4qVYZ95vLWwqyNJCQg7rDsG5wPdze0UaPolH7DUk= +github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= +github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/codahale/hdrhistogram v0.0.0-20160425231609-f8ad88b59a58/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= +github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= +github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= +github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= +github.com/containerd/btrfs v0.0.0-20201111183144-404b9149801e/go.mod h1:jg2QkJcsabfHugurUvvPhS3E08Oxiuh5W/g1ybB4e0E= +github.com/containerd/btrfs v0.0.0-20210316141732-918d888fb676/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= +github.com/containerd/btrfs v1.0.0/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= +github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI= +github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= +github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= +github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= +github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= +github.com/containerd/cgroups v1.0.1 h1:iJnMvco9XGvKUvNQkv88bE4uJXxRQH18efbKo9w5vHQ= +github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= +github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= +github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= +github.com/containerd/console v1.0.0/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= +github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= +github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= +github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.1-0.20191213020239-082f7e3aed57/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.4.4 h1:rtRG4N6Ct7GNssATwgpvMGfnjnwfjnu/Zs9W3Ikzq+M= -github.com/containerd/containerd v1.4.4/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.0-beta.2.0.20200729163537-40b22ef07410/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.1-0.20201117152358-0edc412565dc/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ= +github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU= +github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI= +github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= +github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= +github.com/containerd/containerd v1.5.2/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= +github.com/containerd/containerd v1.5.3/go.mod h1:sx18RgvW6ABJ4iYUw7Q5x7bgFOAB9B6G7+yO0XBc4zw= +github.com/containerd/containerd v1.5.4/go.mod h1:sx18RgvW6ABJ4iYUw7Q5x7bgFOAB9B6G7+yO0XBc4zw= +github.com/containerd/containerd v1.5.7 h1:rQyoYtj4KddB3bxG6SAqd4+08gePNyJjRqvOIfV3rkM= +github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7 h1:6ejg6Lkk8dskcM7wQ28gONkukbQkM4qpj4RnYbpFzrI= +github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo= github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y= +github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= +github.com/containerd/continuity v0.1.0 h1:UFRRY5JemiAhPZrr/uE0n8fMTLcZsUvySPr1+D7pgr8= +github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= +github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= +github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= +github.com/containerd/fifo v0.0.0-20201026212402-0724c46b320c/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= +github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= +github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= +github.com/containerd/fuse-overlayfs-snapshotter v1.0.2/go.mod h1:nRZceC8a7dRm3Ao6cJAwuJWPFiBPaibHiFntRUnzhwU= +github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU= +github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk= github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= +github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= +github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g= +github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= +github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= +github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0= +github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA= +github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow= +github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms= +github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= +github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= +github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= +github.com/containerd/stargz-snapshotter v0.0.0-20201027054423-3a04e4c2c116/go.mod h1:o59b3PCKVAf9jjiKtCc/9hLAd+5p/rfhBfm6aBcTEr4= +github.com/containerd/stargz-snapshotter v0.6.4/go.mod h1:1t0SF1gAHJhCSftWKDLVitvfF3c2qhL5hymG7C50wto= +github.com/containerd/stargz-snapshotter/estargz v0.6.4/go.mod h1:83VWDqHnurTKliEB0YvWMiCfLDwv4Cjj1X9Vk98GJZw= github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= +github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= +github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8= +github.com/containerd/ttrpc v1.0.1/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= +github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= +github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk= +github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg= +github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY7aY= +github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= +github.com/containerd/zfs v0.0.0-20200918131355-0a33824f23a2/go.mod h1:8IgZOBdv8fAgXddBT4dBXJPtxyRsejFIpXoklgxgEjw= +github.com/containerd/zfs v0.0.0-20210301145711-11e8f1707f62/go.mod h1:A9zfAbMlQwE+/is6hi0Xw8ktpL+6glmqZYtevJgaB8Y= +github.com/containerd/zfs v0.0.0-20210315114300-dde8f0fda960/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= +github.com/containerd/zfs v0.0.0-20210324211415-d5c4544f0433/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= +github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= +github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM= +github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8= +github.com/containers/image v3.0.2+incompatible h1:B1lqAE8MUPCrsBLE86J0gnXleeRq8zJnQryhiiGQNyE= +github.com/containers/image v3.0.2+incompatible/go.mod h1:8Vtij257IWSanUQKe1tAeNOm2sRVkSqQTVQ1IlwI3+M= +github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= +github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= +github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= github.com/coocood/freecache v1.1.0 h1:ENiHOsWdj1BrrlPwblhbn4GdAsMymK3pZORJ+bJGAjA= github.com/coocood/freecache v1.1.0/go.mod h1:ePwxCDzOYvARfHdr1pByNct1at3CoKnsipOHwKlNbzI= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= +github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= +github.com/coreos/go-oidc v2.1.0+incompatible h1:sdJrfw8akMnCuUlaZU3tE/uYXFgfqom8DBE9so9EBsM= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-oidc/v3 v3.0.0 h1:/mAA0XMgYJw2Uqm7WKGCsKnjitE/+A0FFbOmiRJm7LQ= +github.com/coreos/go-oidc/v3 v3.0.0/go.mod h1:rEJ/idjfUyfkBit1eI1fvyr+64/g9dcKpAm8MJMesvo= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -189,18 +445,42 @@ github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg= github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= +github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= +github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= +github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8= +github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I= +github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE= -github.com/deislabs/oras v0.11.1 h1:oo2J/3vXdcti8cjFi8ghMOkx0OacONxHC8dhJ17NdJ0= -github.com/deislabs/oras v0.11.1/go.mod h1:39lCtf8Q6WDC7ul9cnyWXONNzKvabEKk+AX+L0ImnQk= -github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= +github.com/denisenkom/go-mssqldb v0.9.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= +github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= +github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= +github.com/dexidp/dex v0.0.0-20210802203454-3fac2ab6bc3b h1:ovHbNjGAQsGEs67tYU6C6ex2D2shDkeZ7pPprx58f2k= +github.com/dexidp/dex v0.0.0-20210802203454-3fac2ab6bc3b/go.mod h1:g64CEwk9b4oLTREOu8mFkjTkDUvyxzWGqhnPwQlfrq8= +github.com/dexidp/dex/api/v2 v2.0.0/go.mod h1:k5arBJT1QYvpsEY3sEd0NXJp3hKWKuUUfzJ3BlcqPdM= +github.com/dgraph-io/badger/v3 v3.2103.2 h1:dpyM5eCJAtQCBcMCZcT4UBZchuTJgCywerHHgmxfxM8= +github.com/dgraph-io/badger/v3 v3.2103.2/go.mod h1:RHo4/GmYcKKh5Lxu63wLEMHJ70Pac2JqZRYGhlyAo2M= +github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= +github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= +github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/docker/cli v20.10.5+incompatible h1:bjflayQbWg+xOkF2WPEAOi4Y7zWhR7ptoPhV/VqLVDE= -github.com/docker/cli v20.10.5+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= +github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/distribution/distribution/v3 v3.0.0-20210804104954-38ab4c606ee3 h1:rEK0juuU5idazw//KzUcL3yYwUU3DIe2OnfJwjDBqno= +github.com/distribution/distribution/v3 v3.0.0-20210804104954-38ab4c606ee3/go.mod h1:gt38b7cvVKazi5XkHvINNytZXgTEntyhtyM3HQz46Nk= +github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/docker/cli v0.0.0-20190925022749-754388324470/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v20.10.0-beta1.0.20201029214301-1d20b15adc38+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v20.10.6+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v20.10.7+incompatible h1:pv/3NqibQKphWZiAskMzdz8w0PRbtTaEB+f6NwdU7Is= +github.com/docker/cli v20.10.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.6.0-rc.1.0.20170726174610-edc3ab29cdff+incompatible h1:357nGVUC8gSpeSc2Axup8HfrfTLLUfWfCsCUhiQSKIg= github.com/docker/distribution v2.6.0-rc.1.0.20170726174610-edc3ab29cdff+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v0.0.0-20180502112750-51a9119f6b81 h1:VnJAjP/7FbY46wddaiKakrc1ogXW+3MlP7cg+S4j4Og= @@ -209,9 +489,16 @@ github.com/docker/docker-credential-helpers v0.6.3 h1:zI2p9+1NQYdnG6sMU26EX4aVGl github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= github.com/docker/go-connections v0.3.1-0.20180212134524-7beb39f0b969 h1:Z8LR4yJgA3vaXpBiUWcWdQZpB9h+dzj4NdLjP9Sfdb0= github.com/docker/go-connections v0.3.1-0.20180212134524-7beb39f0b969/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= +github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= +github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/libnetwork v0.8.0-dev.2.0.20200917202933-d0951081b35f/go.mod h1:93m0aTqz6z+g32wla4l4WxTrdtvBRmVzYRkYvasA5Z8= +github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4= github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= @@ -219,23 +506,25 @@ github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= +github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.11.0+incompatible h1:glyUF9yIYtMHzn8xaKw5rMhdWcwsYV8dZHIq5567/xs= @@ -243,53 +532,71 @@ github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQL github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= -github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= +github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.0.2 h1:+nS9g82KMXccJ/wp0zyRW9ZBHFETmMGtkk+2CTTrW4o= +github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= -github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= github.com/garyburd/redigo v1.6.2 h1:yE/pwKCrbLpLpQICzYTeZ7JsTA/C53wFTJHaEtRqniM= github.com/garyburd/redigo v1.6.2/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/gzip v0.0.1 h1:ezvKOL6jH+jlzdHNE4h9h8q8uMpDQjyl0NN0Jd7jozc= github.com/gin-contrib/gzip v0.0.1/go.mod h1:fGBJBCdt6qCZuCAOwWuFhBB4OOq9EFqlo5dEaFhhu5w= -github.com/gin-contrib/sessions v0.0.3 h1:PoBXki+44XdJdlgDqDrY5nDVe3Wk7wDV/UCOuLP6fBI= -github.com/gin-contrib/sessions v0.0.3/go.mod h1:8C/J6cad3Il1mWYYgtw0w+hqasmpvy25mPkXdOgeB9I= github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= -github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmCsR2Do= github.com/gin-gonic/gin v1.7.2 h1:Tg03T9yM2xa8j6I3Z3oqLaQRSmKvxPd6g/2HJ6zICFA= github.com/gin-gonic/gin v1.7.2/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY= +github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/go-asn1-ber/asn1-ber v1.5.1 h1:pDbRAunXzIUXfx4CB2QJFv5IuPiuoW+sWvr/Us009o8= +github.com/go-asn1-ber/asn1-ber v1.5.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= +github.com/go-bindata/go-bindata v1.0.1-0.20190711162640-ee3c2418e368/go.mod h1:7xCgX1lzlrXPHkfvn3EhumqHkmSlzt8at9q7v0ax19c= +github.com/go-co-op/gocron v1.13.0 h1:BjkuNImPy5NuIPEifhWItFG7pYyr27cyjS6BN9w/D4c= +github.com/go-co-op/gocron v1.13.0/go.mod h1:GD5EIEly1YNW+LovFVx5dzbYVcIc8544K99D8UVRpGo= +github.com/go-critic/go-critic v0.4.1/go.mod h1:7/14rZGnZbY6E38VEGk2kVhoq6itzc1E68facVDK23g= +github.com/go-critic/go-critic v0.4.3/go.mod h1:j4O3D4RoIwRqlZw5jJpx0BNfXWWbpcJoKu5cYSe4YmQ= github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-ldap/ldap/v3 v3.3.0 h1:lwx+SJpgOHd8tG6SumBQZXCmNX51zM8B1cfxJ5gv4tQ= +github.com/go-ldap/ldap/v3 v3.3.0/go.mod h1:iYS1MdmrmceOJ1QOTnRXrIs7i3kloqtmGQjRvjKpyMg= +github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/zapr v0.4.0 h1:uc1uML3hRYL9/ZZPdgHS/n8Nzo+eaYL/Efxkkamf7OM= github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= +github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= @@ -298,17 +605,22 @@ github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2 github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= +github.com/go-openapi/inflect v0.19.0/go.mod h1:lHpZVlpIQqLyKwJ4N+YSc9hchQy/i12fJykb83CRBH4= +github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= -github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.19.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= -github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM= +github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= @@ -317,6 +629,7 @@ github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCs github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= +github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.19.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= @@ -329,20 +642,20 @@ github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pL github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= +github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= github.com/go-openapi/validate v0.19.8/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= @@ -351,18 +664,33 @@ github.com/go-resty/resty/v2 v2.6.0 h1:joIR5PNLM2EFqqESUjCMGXrWmXNHEU9CEiK813oKY github.com/go-resty/resty/v2 v2.6.0/go.mod h1:PwvJS6hvaPkjtjNg9ph+VrSD92bi5Zq73w/BIH7cC3Q= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-sql-driver/mysql v1.5.1-0.20200311113236-681ffa848bae/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4= +github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ= +github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= +github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= +github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg= +github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw= +github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU= +github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk= +github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI= +github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks= +github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc= +github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= +github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= +github.com/go-toolsmith/typep v1.0.2/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= +github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/envy v1.7.1 h1:OQl5ys5MBea7OGCdvPbBJWRgnhC/fGona6QKfvFeau8= -github.com/gobuffalo/envy v1.7.1/go.mod h1:FurDp9+EDPE4aIUS3ZLyD+7/9fpx7YRt/ukY6jIHf0w= github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= @@ -376,52 +704,73 @@ github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSC github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM= github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= -github.com/gobuffalo/logger v1.0.1 h1:ZEgyRGgAm4ZAhAO45YXMs5Fp+bzGLESFewzAVBMKuTg= -github.com/gobuffalo/logger v1.0.1/go.mod h1:2zbswyIUa45I+c+FLXuWl9zSWEiVuthsk8ze5s8JvPs= +github.com/gobuffalo/logger v1.0.3 h1:YaXOTHNPCvkqqA7w05A4v0k2tCdpr+sgFlgINbQ6gqc= +github.com/gobuffalo/logger v1.0.3/go.mod h1:SoeejUwldiS7ZsyCBphOGURmWdwUFXs0J7TCjEhjKxM= github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packd v0.3.0 h1:eMwymTkA1uXsqxS0Tpoop3Lc0u3kTfiMBE6nKtQU4g4= -github.com/gobuffalo/packd v0.3.0/go.mod h1:zC7QkmNkYVGKPw4tHpBQ+ml7W/3tIebgeo1b36chA3Q= +github.com/gobuffalo/packd v1.0.0 h1:6ERZvJHfe24rfFmA9OaoKBdC7+c9sydrytMg8SdFGBM= +github.com/gobuffalo/packd v1.0.0/go.mod h1:6VTc4htmJRFB7u1m/4LeMTWjFoYrUiBkU9Fdec9hrhI= github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= -github.com/gobuffalo/packr/v2 v2.7.1 h1:n3CIW5T17T8v4GGK5sWXLVWJhCz7b5aNLSxW6gYim4o= -github.com/gobuffalo/packr/v2 v2.7.1/go.mod h1:qYEvAazPaVxy7Y7KR0W8qYEE+RymX74kETFqjFoFlOc= +github.com/gobuffalo/packr/v2 v2.8.1 h1:tkQpju6i3EtMXJ9uoF5GT6kB+LMTimDWD8Xvbz6zDVA= +github.com/gobuffalo/packr/v2 v2.8.1/go.mod h1:c/PLlOuTU+p3SybaJATW3H6lX/iK7xEz5OeMf+NnJpg= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godror/godror v0.13.3/go.mod h1:2ouUT4kdhUBk7TAkHWD4SN0CdI0pgEQbo8FVHhbSKWg= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godror/godror v0.24.2/go.mod h1:wZv/9vPiUib6tkoDl+AZ/QLf5YZgMravZ7jxH2eQWAE= +github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= +github.com/gofrs/flock v0.7.3/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= +github.com/gogo/googleapis v1.3.2/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= +github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= +github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -431,71 +780,138 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4-0.20210608040537-544b4180ac70 h1:yxuuMouxXYv9V1HprM9jTODJPGrTrC0FYVtPSnyIXxs= +github.com/golang/snappy v0.0.4-0.20210608040537-544b4180ac70/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4= +github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk= +github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0= +github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8= +github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o= +github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU= +github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU= +github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= +github.com/golangci/golangci-lint v1.23.7/go.mod h1:g/38bxfhp4rI7zeWSxcdIeHTQGS58TCak8FYcyCmavQ= +github.com/golangci/golangci-lint v1.27.0/go.mod h1:+eZALfxIuthdrHPtfM7w/R3POJLjHDfJJw8XZl9xOng= +github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU= +github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= +github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o= +github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA= +github.com/golangci/misspell v0.3.5/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA= +github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI= +github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4= +github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4= +github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ= github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= -github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k= +github.com/gomodule/redigo v1.8.2/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= +github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/crfs v0.0.0-20191108021818-71d77da419c9/go.mod h1:etGhoOqfwPkooV6aqoX3eBGQOJblqdoc9XvWOeuxpPw= +github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= +github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-containerregistry v0.0.0-20191010200024-a3d713f9b7f8/go.mod h1:KyKXa9ciM8+lgMXwOVsXi7UxGrsf9mM61Mzs+xKUrKE= +github.com/google/go-containerregistry v0.1.2/go.mod h1:GPivBPgdAyd2SU+vf6EpsgOtWDuPqjW0hJZt4rNdTZ4= +github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= +github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-github/v28 v28.1.1/go.mod h1:bsqJWQX05omyWVmc00nEUql9mhQyv38lDZ8kPZcQVoM= github.com/google/go-github/v29 v29.0.2 h1:opYN6Wc7DOz7Ku3Oh4l7prmkOMwEcQxpFtxdU8N8Pts= github.com/google/go-github/v29 v29.0.2/go.mod h1:CHKiKKPHJ0REzfwc14QMklvtHwCveD0PxlMjLlzAM5E= github.com/google/go-github/v35 v35.3.0 h1:fU+WBzuukn0VssbayTT+Zo3/ESKX9JYWjbZTLOTEyho= github.com/google/go-github/v35 v35.3.0/go.mod h1:yWB7uCcVWaUbUP74Aq3whuMySRMatyRmq5U9FTNlbio= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/go-replayers/grpcreplay v0.1.0/go.mod h1:8Ig2Idjpr6gifRd6pNVggX6TC1Zw6Jx74AKp7QNH2QE= +github.com/google/go-replayers/httpreplay v0.1.0/go.mod h1:YKZViNhiGgqdBlUbI2MwGpq4pXxNmhJLPHQ7cv2b5no= +github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= +github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210715191844-86eeefc3e471/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/rpmpack v0.0.0-20191226140753-aa36bfddb3a0/go.mod h1:RaTPr0KUf2K7fnZYLNDrr8rxAamWs3iNywJLtQ2AzBg= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/wire v0.3.0/go.mod h1:i1DMg/Lu8Sz5yYl25iOdmc5CT5qusaa+zmRWs16741s= +github.com/google/wire v0.4.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU= +github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= +github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww= +github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.2.2/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= +github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= +github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= +github.com/gookit/color v1.2.4/go.mod h1:AhIE+pS6D4Ql0SQWbBeXPHw7gY0/sjHoA4s/n1KB7xg= +github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= +github.com/goreleaser/goreleaser v0.136.0/go.mod h1:wiKrPUeSNh6Wu8nUHxZydSOVQ/OZvOaO7DTtFqie904= +github.com/goreleaser/nfpm v1.2.1/go.mod h1:TtWrABZozuLOttX2uDlYyECfQX7x5XYkVxhjYcR6G9w= +github.com/goreleaser/nfpm v1.3.0/go.mod h1:w0p7Kc9TAUgWMyrub63ex3M2Mgw88M4GZXoTq5UCb40= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= -github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= -github.com/gorilla/sessions v1.1.1/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w= -github.com/gorilla/sessions v1.1.3 h1:uXoZdcdA5XdXF3QzuSlheVRUvjl+1rKY7zBXL68L9RU= -github.com/gorilla/sessions v1.1.3/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= +github.com/gostaticanalysis/analysisutil v0.0.3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= github.com/gotestyourself/gotestyourself v2.2.0+incompatible h1:AQwinXlbQR2HvPjQZOmDhRqsv5mZf+Jb1RnSLxcqZcI= @@ -505,14 +921,23 @@ github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f h1:ShTPMJQes6t github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.2.0/go.mod h1:mJzapYve32yjrKlk9GbyCZHuPgZsrbyIbyKhSzOpg6s= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/grpc-ecosystem/grpc-gateway v1.6.2/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.9.2/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw= +github.com/hanwen/go-fuse v1.0.0/go.mod h1:unqXarDXqzAk0rt98O2tVndEPIpUgLD9+rwFisZH3Ok= +github.com/hanwen/go-fuse/v2 v2.0.3/go.mod h1:0EQM6aH2ctVpvZ6a+onrQ/vaykxh2GH7hy3e13vzTUY= +github.com/hanwen/go-fuse/v2 v2.1.0/go.mod h1:oRyA5eK+pvJyv5otpO/DgccS8y/RvYMaO00GgRLGryc= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= @@ -520,10 +945,14 @@ github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtng github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.6.4/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/hashicorp/go-retryablehttp v0.6.6/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs= github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= @@ -534,44 +963,73 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= +github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hashicorp/uuid v0.0.0-20160311170451-ebb0a03e909c/go.mod h1:fHzc09UnyJyqyW+bFuq864eh+wC7dj65aXmXLRe5to0= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huandu/xstrings v1.3.1 h1:4jgBlKK6tLKFvO8u5pmYjG91cqytmDCDvGh7ECVFfFs= github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= +github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/huaweicloud/huaweicloud-sdk-go-v3 v0.0.50 h1:nQ1QIkLxAoFsYyjvpQH3gVIy3mHfI0WDYEWdmwwxSG0= github.com/huaweicloud/huaweicloud-sdk-go-v3 v0.0.50/go.mod h1:Pp3sd2tx3j9qC7Ij6jGh5phZwTrI+/HUBK90f3Cn2CI= -github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/ishidawataru/sctp v0.0.0-20191218070446-00ab2ac2db07/go.mod h1:co9pwDoBCm1kGxawmb4sPq0cSIOOWNPT4KnHotMP1Zg= +github.com/ishidawataru/sctp v0.0.0-20210226210310-f2269e66cdee/go.mod h1:co9pwDoBCm1kGxawmb4sPq0cSIOOWNPT4KnHotMP1Zg= +github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= +github.com/jaguilar/vt100 v0.0.0-20150826170717-2703a27b14ea/go.mod h1:QMdK4dGB3YhEW2BmA1wgGpPYI3HZy/5gD705PXKUVSg= +github.com/jarcoal/httpmock v1.0.5/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= github.com/jasonlvhit/gocron v0.0.0-20171226191223-3c914c8681c3 h1:s2ARkZWJfkyjrUi8Dj0mMbc1inxeOnIrO9zc3oQX8iU= github.com/jasonlvhit/gocron v0.0.0-20171226191223-3c914c8681c3/go.mod h1:rwi/esz/h+4oWLhbWWK7f6dtmgLzxeZhnwGr7MCsTNk= +github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a/go.mod h1:xRskid8CManxVta/ALEhJha/pweKBaVG6fWgc0yH25s= +github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg= +github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.2 h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI= github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0= +github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0= +github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jmoiron/sqlx v1.3.1 h1:aLN7YINNZ7cYOPK3QC83dbM6KT0NMqVMw961TqrejlE= github.com/jmoiron/sqlx v1.3.1/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= -github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= +github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jonboulle/clockwork v0.2.0/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -590,107 +1048,177 @@ github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uia github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= -github.com/kidstuff/mongostore v0.0.0-20181113001930-e650cd85ee4b/go.mod h1:g2nVr8KZVXJSS97Jo8pJ0jgq29P6H7dG0oplUA86MQw= +github.com/karrick/godirwalk v1.15.8 h1:7+rWAZPn9zuRxaIqqT8Ohs2Q2Ac0msBqwRdxNCr2VVs= +github.com/karrick/godirwalk v1.15.8/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.9.5 h1:U+CaK85mrNNb4k8BNOfgJtJ/gr6kswUCFj6miSzVC6M= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= +github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kortschak/utter v1.0.1/go.mod h1:vSmSjbyrlKjjsL71193LmzBOKgwePk9DH6uFaWHIInc= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= -github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E= github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= -github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= -github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= -github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= +github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/maratori/testpackage v1.0.1/go.mod h1:ddKdw+XG0Phzhx8BFDTKgpWP4i7MpApTE5fXSKAqwDU= +github.com/markbates/errx v1.1.0 h1:QDFeR+UP95dO12JgW+tgi2UVfo0V8YBHiUIOaeBPiEI= +github.com/markbates/errx v1.1.0/go.mod h1:PLa46Oex9KNbVDZhKel8v1OT7hD5JZ2eI7AHhA0wswc= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= +github.com/markbates/oncer v1.0.0 h1:E83IaVAHygyndzPimgUYJjbshhDTALZyXxvk9FOlQRY= +github.com/markbates/oncer v1.0.0/go.mod h1:Z59JA581E9GP6w96jai+TGqafHPW+cPfRxz2aSZ0mcI= github.com/markbates/pkger v0.17.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI= +github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= -github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= +github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= +github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s= +github.com/mattermost/xml-roundtrip-validator v0.1.0/go.mod h1:qccnGMcpgwcNaBnxqpJpWWUiPNr5H3O8eDgGV9gT5To= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-oci8 v0.0.7/go.mod h1:wjDx6Xm9q7dFtHJvIlrI99JytznLw5wQ4R+9mNXJwGI= +github.com/mattn/go-oci8 v0.1.1/go.mod h1:wjDx6Xm9q7dFtHJvIlrI99JytznLw5wQ4R+9mNXJwGI= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= +github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/mattn/go-shellwords v1.0.11/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= -github.com/mattn/go-sqlite3 v1.12.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/go-sqlite3 v1.14.8 h1:gDp86IdQsN/xWjIEmr9MF6o9mpksUgh0fu+9ByFxzIU= +github.com/mattn/go-sqlite3 v1.14.8/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= +github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/mattomatic/dijkstra v0.0.0-20130617153013-6f6d134eb237 h1:acuCHBjzG7MFTugvx3buC4m5rLDLaKC9J8C9jtlraRc= github.com/mattomatic/dijkstra v0.0.0-20130617153013-6f6d134eb237/go.mod h1:UOnLAUmVG5paym8pD3C4B9BQylUDC2vXFJJpT7JrlEA= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/memcachier/mc v2.0.1+incompatible/go.mod h1:7bkvFE61leUBvXz+yxsOnGBQSZpBSPIMUQSmmSHvuXc= +github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/cli v1.1.2/go.mod h1:6iaV0fGdElS6dPBx0EApTxHrcWvmJphyh2n8YBLPPZ4= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= -github.com/mitchellh/copystructure v1.1.1 h1:Bp6x9R1Wn16SIz3OfeDr0b7RnCG2OB66Y7PQyC/cvq4= github.com/mitchellh/copystructure v1.1.1/go.mod h1:EBArHfARyrSWO/+Wyr9zwEkc6XMFB9XyNgFNmRkZZU4= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/hashstructure v1.0.0/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.3.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/mitchellh/reflectwalk v1.0.1 h1:FVzMWA5RllMAKIdUSC8mdWo3XtwoecrH79BY70sEEpE= github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/mittwald/go-helm-client v0.8.0 h1:k8yMvoWE7C+jJgw6iodJ0U5nY4hVpLU20VIpR+FfyJk= -github.com/mittwald/go-helm-client v0.8.0/go.mod h1:+Ks5OpzB0hJw91W8o/8G+FmjUtjrIXxPuJjE87XG/G0= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mittwald/go-helm-client v0.8.4 h1:2T/MUkgUjeP+h2P0YYyuwXN8jB/l90oQ12JQ5+9h9tI= +github.com/mittwald/go-helm-client v0.8.4/go.mod h1:GtL4+taCH4ADXO+aimCEQ5EtPRSGPSHcT6d/fB3coFk= +github.com/moby/buildkit v0.8.1/go.mod h1:/kyU1hKy/aYCuP39GZA9MaKioovHku57N6cqlKZIaiQ= +github.com/moby/buildkit v0.9.1 h1:6noq8jvkaRs3OQGTIDf5U5Etkgq6zsPmQHGWi5yhh88= +github.com/moby/buildkit v0.9.1/go.mod h1:oVZKk3TMm0MlDx7XxnlF0wKmcpyrzOs9GEp0VXKWFPk= +github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= +github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/moby/sys/mount v0.1.0/go.mod h1:FVQFLDRWwyBjDTBNQXDlWnSFREqOo3OKX9aqhmeoo74= +github.com/moby/sys/mount v0.1.1/go.mod h1:FVQFLDRWwyBjDTBNQXDlWnSFREqOo3OKX9aqhmeoo74= +github.com/moby/sys/mount v0.2.0/go.mod h1:aAivFE2LB3W4bACsUXChRHQ0qKWsetY4Y9V7sxOougM= +github.com/moby/sys/mountinfo v0.1.0/go.mod h1:w2t2Avltqx8vE7gX5l+QiBKxODu2TX0+Syr3h52Tw4o= +github.com/moby/sys/mountinfo v0.1.3/go.mod h1:w2t2Avltqx8vE7gX5l+QiBKxODu2TX0+Syr3h52Tw4o= +github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/sys/mountinfo v0.4.1 h1:1O+1cHA1aujwEwwVMa2Xm2l+gIpUHyd3+D+d7LZh1kM= +github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= +github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= +github.com/moby/term v0.0.0-20200915141129-7f0af18e79f2/go.mod h1:TjQg8pa4iejrUrjiz0MCtMV38jdMNW4doKSiBrEvCQQ= +github.com/moby/term v0.0.0-20201110203204-bea5bbe245bf/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 h1:yH0SvLzcbZxcJXho2yh7CqdENGMQe73Cw3woZBpPli0= github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -698,20 +1226,19 @@ github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/ github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= +github.com/mozilla/tls-observatory v0.0.0-20200317151703-4fa42e1c2dee/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= +github.com/mrunalp/fileutils v0.0.0-20200520151820-abd8a0e76976/go.mod h1:x8F1gnqOkIEiO4rqoeEEEqQbo7HjGMTvyoq3gej4iT0= +github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= -github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= -github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= -github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= -github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/nakabonne/nestif v0.3.0/go.mod h1:dI314BppzXjJ4HsCnbo7XzrJHPszZsjnk5wEBSYHI2c= +github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= +github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nsqio/go-nsq v1.0.7 h1:O0pIZJYTf+x7cZBA0UMY8WxFG79lYTURmWzAAh48ljY= github.com/nsqio/go-nsq v1.0.7/go.mod h1:XP5zaUs3pqf+Q71EqUJs3HYfBIqfK6G83WQMdNN+Ito= @@ -720,47 +1247,70 @@ github.com/nwaples/rardecode v1.0.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWk github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= -github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/olekukonko/tablewriter v0.0.2/go.mod h1:rSAaSIOAGT9odnlyGlUfAJaoc5w2fSBUmeGDbRWPxyQ= github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= +github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU= github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v0.1.1 h1:GlxAyO6x8rfZYN9Tt0Kti5a/cP41iuiO2yYT0IJGY8Y= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc10/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc92/go.mod h1:X1zlU4p7wOlX4+WRCz+hvlRv8phdL7UqbYD+vQwNMmE= +github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= +github.com/opencontainers/runc v1.0.2 h1:opHZMaswlyxz1OuGpBE53Dwe4/xF7EZTY0A2L/FpCOg= +github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= +github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/selinux v1.6.0 h1:+bIAS/Za3q5FTwWym4fTB0vObnfCf3G/NC7K6Jx62mY= +github.com/opencontainers/runtime-spec v1.0.3-0.20200728170252-4d89ac9fbff6/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= -github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= -github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= -github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= +github.com/opencontainers/selinux v1.8.2 h1:c4ca10UMgRcvZ6h0K4HtS15UaVSBEaE+iln2LVpAuGc= +github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= +github.com/opentracing-contrib/go-stdlib v1.0.0/go.mod h1:qtI1ogk+2JhVPIXVc6q+NHziSmy2W5GbdQZFUHADCBU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= +github.com/openzipkin/zipkin-go v0.1.3/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= -github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/otiai10/copy v1.6.0 h1:IinKAryFFuPONZ7cm6T6E2QX/vcJwSnlaA5lfoaXIiQ= github.com/otiai10/copy v1.6.0/go.mod h1:XWfuS3CrI0R6IE0FbgHsEazaXO8G0LpMp9o8tos0x4E= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= @@ -768,35 +1318,42 @@ github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6 github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= github.com/otiai10/mint v1.3.2 h1:VYWnrP5fXmz1MXvjuUvcBrXSjGE6xjON+axB/UrpO3E= github.com/otiai10/mint v1.3.2/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= -github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= -github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= +github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= +github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= +github.com/pelletier/go-toml v1.9.1/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= +github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d/go.mod h1:3OzsM7FXDQlpCiw2j81fOmAwQLnZnLGXVKUzeKQXIAw= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= -github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= +github.com/pkg/profile v1.5.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= @@ -804,56 +1361,84 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1: github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/quasoft/memstore v0.0.0-20180925164028-84a050167438/go.mod h1:wTPjTepVu7uJBYgZ0SdWHQlIas582j6cn2jgk4DDdlg= +github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= +github.com/quasilyte/go-ruleguard v0.1.2-0.20200318202121-b00d7a75d3d8/go.mod h1:CGFX09Ci3pq9QZdj86B+VGIdNj4VyCo2iPOGS9esB/k= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/rfyiamcool/cronlib v1.0.0 h1:PNTF7pgtbL7TedrqnC6syhHwj4d1bCKxH3XmPqqgxvg= github.com/rfyiamcool/cronlib v1.0.0/go.mod h1:i7AVVUhM/kkNcC/Ayq0XOmyCJCcqa/FlflyYQMm5QWE= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.4.0 h1:LUa41nrWTQNGhzdsZ5lTnkwbNjj6rXTdazA1cSdjkOY= -github.com/rogpeppe/go-internal v1.4.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351 h1:HXr/qUllAWv9riaI4zh2eXWKmCSDqVS/XH1MRHLKRwk= -github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351/go.mod h1:DCgfY80j8GYL7MLEfvcpSFvjD0L5yZq/aZUJmhZklyg= +github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rubenv/sql-migrate v0.0.0-20210614095031-55d5740dbbcc h1:BD7uZqkN8CpjJtN/tScAKiccBikU4dlqe/gNrkRaPY4= +github.com/rubenv/sql-migrate v0.0.0-20210614095031-55d5740dbbcc/go.mod h1:HFLT6i9iR4QBOF5rdCyjddC9t59ArqWJV2xx+jwcCMo= +github.com/rubiojr/go-vhd v0.0.0-20160810183302-0bfd3b39853c/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto= +github.com/russellhaering/goxmldsig v1.1.0/go.mod h1:QK8GhXPB3+AfuCrfo0oRISa9NfzeCpWmxeGnqEpDF9o= github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryancurrah/gomodguard v1.0.4/go.mod h1:9T/Cfuxs5StfsocWr4WzDL36HqnX0fVb9d5fSEaLhoE= +github.com/ryancurrah/gomodguard v1.1.0/go.mod h1:4O8tr7hBODaGE6VIhfJDHcwzh5GUccKSJBU0UMXJFVM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= +github.com/sassoftware/go-rpmutils v0.0.0-20190420191620-a8f1baeba37b/go.mod h1:am+Fp8Bt506lA3Rk3QCmSqmYmLMnPDhdDUcosQCAx+I= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= +github.com/securego/gosec v0.0.0-20200103095621-79fbf3af8d83/go.mod h1:vvbZ2Ae7AzSq3/kywjUDxSNq2SJ27RxCz2un0H3ePqE= +github.com/securego/gosec v0.0.0-20200401082031-e946c8c39989/go.mod h1:i9l/TNj+yDFh9SZXUTvspXTjbFXgZGP/UvhU1S65A4A= +github.com/securego/gosec/v2 v2.3.0/go.mod h1:UzeVyUXbxukhLeHKV3VVqo7HdoQR9MrRfFmZYotn8ME= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/serialx/hashring v0.0.0-20190422032157-8b2912629002/go.mod h1:/yeG0My1xr/u+HZrFQ1tOQQQQrOawfyMUH13ai5brBc= +github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada h1:WokF3GuxBeL+n4Lk4Fa8v9mbdjlrl7bHuneF4N1bk2I= +github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= +github.com/shirou/gopsutil/v3 v3.22.5 h1:atX36I/IXgFiB81687vSiBI5zrMsxcIBkP9cQMJQoJA= +github.com/shirou/gopsutil/v3 v3.22.5/go.mod h1:so9G9VzeHt/hsd0YwqprnjHnfARAUktauykSbr+y2gA= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= @@ -862,29 +1447,39 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= +github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= +github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE= +github.com/sourcegraph/go-diff v0.5.3/go.mod h1:v9JDtjCE4HHHCZGId75rg8gkKKa98RVjBcBGsVmMmak= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= -github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= +github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= +github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -893,24 +1488,29 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= +github.com/spf13/viper v1.6.1/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44= +github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= +github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= github.com/stevvooe/resumable v0.0.0-20180830230917-22b14a53ba50 h1:4bT0pPowCpQImewr+BjzfUKcuFW+KVyB8d1OF3b6oTI= github.com/stevvooe/resumable v0.0.0-20180830230917-22b14a53ba50/go.mod h1:1pdIZTAHUz+HDKDVZ++5xg/duPlhKAIzw9qy42CWYp4= -github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14 h1:PyYN9JH5jY9j6av01SpfRMb+1DWg/i3MbGOKPxJ2wjM= @@ -919,10 +1519,39 @@ github.com/swaggo/gin-swagger v1.3.0 h1:eOmp7r57oUgZPw2dJOjcGNMse9cvXcI4tTqBcnZt github.com/swaggo/gin-swagger v1.3.0/go.mod h1:oy1BRA6WvgtCp848lhxce7BnWH4C8Bxa0m5SkWx+cS0= github.com/swaggo/swag v1.5.1 h1:2Agm8I4K5qb00620mHq0VJ05/KT4FtmALPIcQR9lEZM= github.com/swaggo/swag v1.5.1/go.mod h1:1Bl9F/ZBpVWh22nY0zmYyASPO1lI/zIwRDrpZU+tv8Y= +github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= +github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= +github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM= +github.com/tdakkota/asciicheck v0.0.0-20200416200610-e657995f937b/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM= +github.com/tetafro/godot v0.3.7/go.mod h1:/7NLHhv08H1+8DNj0MElpAACw1ajsCuf3TKNQxA5S+0= +github.com/tetafro/godot v0.4.2/go.mod h1:/7NLHhv08H1+8DNj0MElpAACw1ajsCuf3TKNQxA5S+0= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= +github.com/timakin/bodyclose v0.0.0-20200424151742-cb6215831a94/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= +github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= +github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= +github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= +github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= +github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw= +github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= +github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o= +github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tommy-muehle/go-mnd v1.1.1/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig= +github.com/tommy-muehle/go-mnd v1.3.1-0.20200224220436-e6f9a994e8fa/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig= +github.com/tonistiigi/fsutil v0.0.0-20201103201449-0834f99b7b85/go.mod h1:a7cilN64dG941IOXfhJhlH0qB92hxJ9A1ewrdUmJ6xo= +github.com/tonistiigi/fsutil v0.0.0-20210609172227-d72af97c0eaf/go.mod h1:lJAxK//iyZ3yGbQswdrPTxugZIDM7sd4bEsD0x3XMHk= +github.com/tonistiigi/go-actions-cache v0.0.0-20211002214948-4d48f2ff622a/go.mod h1:YiIBjH5gP7mao3t0dBrNNBGuKYdeJmcAJjYLXr43k6A= +github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea/go.mod h1:WPnis/6cRcDZSUvVmezrxJPkiO87ThFYsoUiMwWNDJk= +github.com/tonistiigi/vt100 v0.0.0-20210615222946-8066bb97264f/go.mod h1:ulncasL3N9uLrVann0m+CDlJKWsIAP34MPcOJF6VRvc= +github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go v1.1.13 h1:nB3O5kBSQGjEQAcfe1aLUYuxmXdFKmYgBZhY32rQb6Q= @@ -932,14 +1561,33 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/ugorji/go/codec v1.1.13 h1:013LbFhocBoIqgHeIHKlV4JWYhqogATYWZhIcH0WHn4= github.com/ugorji/go/codec v1.1.13/go.mod h1:oNVt3Dq+FO91WNQ/9JnHKQP2QJxTzoN7wCBFCq1OeuU= -github.com/ulikunitz/xz v0.5.6 h1:jGHAfXawEGZQ3blwU5wnWKQJvAraT7Ftq9EXjnXYgt8= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= +github.com/ulikunitz/xz v0.5.7 h1:YvTNdFzX6+W5m9msiYg/zpkSURPPtOlzbqYjrFn7Yt4= +github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= +github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA= +github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/uudashr/gocognit v1.0.1/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= +github.com/valyala/quicktemplate v1.2.0/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/vdemeester/k8s-pkg-credentialprovider v1.17.4/go.mod h1:inCTmtUdr5KJbreVojo06krnTgaeAz/Z7lynpPk/Q2c= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= -github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243 h1:R43TdZy32XXSXjJn7M/HhALJ9imq6ztLnChfYJpVDnM= +github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= +github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= +github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= +github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= +github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= +github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= +github.com/xanzy/go-gitlab v0.31.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug= +github.com/xanzy/go-gitlab v0.32.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug= github.com/xanzy/go-gitlab v0.50.0 h1:t7IoYTrnLSbdEZN7d8X/5zcr+ZM4TZQ2mXa8MqWlAZQ= github.com/xanzy/go-gitlab v0.50.0/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= @@ -950,6 +1598,7 @@ github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= @@ -960,13 +1609,20 @@ github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6Ut github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= +github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= github.com/yvasiyarov/go-metrics v0.0.0-20150112132944-c25f46c4b940 h1:p7OofyZ509h8DmPLh8Hn+EIIZm/xYhdZHJ9GnXHdr6U= github.com/yvasiyarov/go-metrics v0.0.0-20150112132944-c25f46c4b940/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= +github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= github.com/yvasiyarov/gorelic v0.0.7 h1:4DTF1WOM2ZZS/xMOkTFBOcb6XiHu/PKn3rVo6dbewQE= github.com/yvasiyarov/gorelic v0.0.7/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= +github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= github.com/yvasiyarov/newrelic_platform_go v0.0.0-20160601141957-9c099fbc30e9 h1:AsFN8kXcCVkUFHyuzp1FtYbzp1nCO/H6+1uPSGEyPzM= github.com/yvasiyarov/newrelic_platform_go v0.0.0-20160601141957-9c099fbc30e9/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= @@ -974,74 +1630,120 @@ github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wK go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= +go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= +go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= +go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= +go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= +go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.5.0 h1:REddm85e1Nl0JPXGGhgZkgJdG/yOe6xvpXUcYK5WLt0= go.mongodb.org/mongo-driver v1.5.0/go.mod h1:boiGPFqyBs5R0R5qf2ErokGRekMfwn+MqKaUyHs7wy0= -go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= +go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0= +go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= +go.opencensus.io v0.19.1/go.mod h1:gug0GbSHa8Pafr0d2urOSgoXHZ6x/RUlaiT0d9pqb4A= +go.opencensus.io v0.19.2/go.mod h1:NO/8qkisMZLZ1FCsKNqtJPwc8/TaclWyY0B6wcYNg9M= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= +go.opentelemetry.io/contrib v0.21.0/go.mod h1:EH4yDYeNoaTqn/8yCWQmfNB78VHfGX2Jt2bvnvzBlGM= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.21.0/go.mod h1:Vm5u/mtkj1OMhtao0v+BGo2LUoLCgHYXvRmj0jWITlE= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.21.0/go.mod h1:a9cocRplhIBkUAJmak+BPDx+LVL7cTmqUPB0uBcTA4k= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.21.0/go.mod h1:JQAtechjxLEL81EjmbRwxBq/XEzGaHcsPuDHAx54hg4= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= +go.opentelemetry.io/otel v1.0.0-RC1/go.mod h1:x9tRa9HK4hSSq7jf2TKbqFbtt58/TGk0f9XiEYISI1I= +go.opentelemetry.io/otel/exporters/jaeger v1.0.0-RC1/go.mod h1:FXJnjGCoTQL6nQ8OpFJ0JI1DrdOvMoVx49ic0Hg4+D4= go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.0.0-RC1/go.mod h1:FliQjImlo7emZVjixV8nbDMAa4iAkcWTE9zzSEOiEPw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.0.0-RC1/go.mod h1:cDwRc2Jrh5Gku1peGK8p9rRuX/Uq2OtVmLicjlw2WYU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.0.0-RC1/go.mod h1:OYKzEoxgXFvehW7X12WYT4/a2BlASJK9l7RtG4A91fg= +go.opentelemetry.io/otel/internal/metric v0.21.0/go.mod h1:iOfAaY2YycsXfYD4kaRSbLx2LKmfpKObWBEv9QK5zFo= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/metric v0.21.0/go.mod h1:JWCt1bjivC4iCrz/aCrM1GSw+ZcvY44KCbaeeRhzHnc= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= +go.opentelemetry.io/otel/oteltest v1.0.0-RC1/go.mod h1:+eoIG0gdEOaPNftuy1YScLr1Gb4mL/9lpDkZ0JjMRq4= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/sdk v1.0.0-RC1/go.mod h1:kj6yPn7Pgt5ByRuwesbaWcRLA+V7BSDg3Hf8xRvsvf8= go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= +go.opentelemetry.io/otel/trace v1.0.0-RC1/go.mod h1:86UHmyHWFEtWjfWPSbu0+d0Pf9Q6e1U+3ViBOc+NXAg= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/proto/otlp v0.9.0/go.mod h1:1vKfU9rv61e9EVGthD1zNvUbiwPcimSsOPU9brfSHJg= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= +gocloud.dev v0.19.0/go.mod h1:SmKwiR8YwIMMJvQBKLsC3fHNyMwXLw3PMDO+VVteJMI= +golang.org/x/build v0.0.0-20190314133821-5284462c4bec/go.mod h1:atTaCNAy0f16Ah5aV1gMSwgiKVHwu/JncqDpuRr7lS4= +golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191122220453-ac88ee75c92c/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= +golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 h1:/pEO3GD/ABYAjuakUS6xSEmmlyVS4kxBNkeA9tLJiTI= +golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= @@ -1052,7 +1754,9 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -1062,6 +1766,7 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= @@ -1073,13 +1778,21 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1095,11 +1808,15 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1107,22 +1824,53 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200505041828-1ed23360d12c/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/oauth2 v0.0.0-20180724155351-3d292e4d0cdc/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914 h1:3B43BWw0xEBsLZ/NO1VALz6fppU3481pik+2Ksv45z8= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1130,20 +1878,26 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181218192612-074acd46bca6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1154,58 +1908,102 @@ golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190515120540-06a5c4944438/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190620070143-6f217b454f45/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200917073148-efd3b9a0ff20/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201013081832-0aaa2718063a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210313202042-bd2e13477e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2 h1:c8PlLMqBbOHoqtjteWm5/kbe6rNY2pbRfbIMVnepueo= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27 h1:XDXtA5hveEEV8JB2l7nhMTp3t3cHp9ZpwcdjqyEWLlo= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1219,25 +2017,38 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181219222714-6e267b5cc78e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190221204921-83362c3779f5/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606050223-4d9ae51c2468/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -1248,37 +2059,64 @@ golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= +golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191004055002-72853e10c5a3/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113232020-e2727e816f5a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200102140908-9497f49d5709/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204192400-7124308813f3/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200308013534-11ec41452d41/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200324003944-a576cf524670/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200331202046-9d5940d49312/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200414032229-332987a829c3/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200422022333-3d57cf2e726e/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200502202811-ed308ab3e770/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1288,8 +2126,19 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1N golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= -google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= +gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= +google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.0.0-20181220000619-583d854617af/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.2.0/go.mod h1:IfRCZScioGtypHNTlz3gFk67J8uePVW7uDTBzXuIkhU= +google.golang.org/api v0.3.0/go.mod h1:IuvZyQh8jgscv8qWfQ4ABd8m7hEudgBFM/EdhA3BnXw= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.5.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4= +google.golang.org/api v0.6.1-0.20190607001116-5213b8090861/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1298,7 +2147,25 @@ google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.25.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.52.0 h1:m5FLEd6dp5CU1F0tMWyqDi2XjchviIz8ntzOSz7w8As= +google.golang.org/api v0.52.0/go.mod h1:Him/adpjt0sxtkWViy0b6xyKW/SD71CwdJ7HqJo7SrU= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1306,14 +2173,22 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181219182458-5a97ab628bfb/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190508193815-b515fa19cec8/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190620144150-6af8c5fc6601/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= @@ -1322,34 +2197,78 @@ google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200527145253-8367513e4ece/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210721163202-f1cecdd8b78a/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210722135532-667f2b7c528f h1:YORWxaStkWBnWgELOHTmDrqNlFXuVGEbhwbB5iK94bQ= +google.golang.org/genproto v0.0.0-20210722135532-667f2b7c528f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0 h1:Klz8I9kdtkIN6EpHHUOMLCYhTn/2WAe5a0s1hcBkdTI= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1361,36 +2280,50 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= +gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/gcfg.v1 v1.2.0/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= -gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= +gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE= +gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= gopkg.in/gorp.v1 v1.7.2 h1:j3DWlAyGVv8whO7AcIWznQ2Yj7yJkn34B8s63GViAAw= gopkg.in/gorp.v1 v1.7.2/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.56.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mholt/archiver.v3 v3.1.1 h1:sLb5X5fPPc9oU6YatdC+vbY46irElT3kPR0wOS7bjR4= gopkg.in/mholt/archiver.v3 v3.1.1/go.mod h1:EwJV1tr6GQdNZ5IFFVakh0aYAjGLVjqvFVL/b8ZVdMw= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= +gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/warnings.v0 v0.1.1/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1403,73 +2336,147 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/mysql v1.1.2 h1:OofcyE2lga734MxwcCW9uB4mWNXMr50uaGRVwQL2B0M= +gorm.io/driver/mysql v1.1.2/go.mod h1:4P/X9vSc3WTrhTLZ259cpFd6xKNYiSSdSZngkSBGIMM= +gorm.io/gorm v1.21.12/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= +gorm.io/gorm v1.21.16 h1:YBIQLtP5PLfZQz59qfrq7xbrK7KWQ+JsXXCH/THlMqs= +gorm.io/gorm v1.21.16/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= -helm.sh/helm/v3 v3.6.2 h1:7YbEhLC6AUgmcDoh5BGFUGNtR/o43pr0AIelSNjhuFU= -helm.sh/helm/v3 v3.6.2/go.mod h1:mIIus8EOqj+obtycw3sidsR4ORr2aFDmXMSI3k+oeVY= +grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= +helm.sh/helm/v3 v3.7.0/go.mod h1:DajHtQTe8KrjNmvy5gxWkosFKaADrS3uRS5EkDtsmI4= +helm.sh/helm/v3 v3.7.2 h1:xn1OxcZEpgKpp4CCpPz1KKUyb9gAtTouXV2E3S8ChYQ= +helm.sh/helm/v3 v3.7.2/go.mod h1:UXuiAn0+FfBpqbiMuwWt8/aAKkfJvnWLBJ6f4HcFs0M= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20180920025451-e3ad64cb4ed3/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.5/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +istio.io/api v0.0.0-20220304035241-8c47cbbea144 h1:JIRhhxQ3LZKOmx2PXHO9xfcnT3mlHLmcewZtpTaAtoo= +istio.io/api v0.0.0-20220304035241-8c47cbbea144/go.mod h1:lavaUNsnT7RGyMFNOGgV5XvOgP3fkTSZkxP/0H/ISt4= +istio.io/client-go v0.0.0-20220304040955-30b642d5ba34 h1:MCFkv6odq6KRBmTM5+NlyNnt7fbYN+yMfOOhin745bQ= +istio.io/client-go v0.0.0-20220304040955-30b642d5ba34/go.mod h1:rKTRGbzRHY/lb3VCCIF4wwTwvkW000acWEcseSy3V9M= +istio.io/gogo-genproto v0.0.0-20210113155706-4daf5697332f h1:9710FpGLvIJ1GGEbpuTh1smVBv+r8cJfR3G82ouSxIQ= +istio.io/gogo-genproto v0.0.0-20210113155706-4daf5697332f/go.mod h1:6BwTZRNbWS570wHX/uR1Wqk5e0157TofTAUMzT7N4+s= k8s.io/api v0.21.0 h1:gu5iGF4V6tfVCQ/R+8Hc0h7H1JuEhzyEi9S4R5LM8+Y= k8s.io/api v0.21.0/go.mod h1:+YbrhBBGgsxbF6o6Kj4KJPJnBmAKuXDeS3E18bgHNVU= k8s.io/apiextensions-apiserver v0.21.0 h1:Nd4uBuweg6ImzbxkC1W7xUNZcCV/8Vt10iTdTIVF3hw= k8s.io/apiextensions-apiserver v0.21.0/go.mod h1:gsQGNtGkc/YoDG9loKI0V+oLZM4ljRPjc/sql5tmvzc= k8s.io/apimachinery v0.21.0 h1:3Fx+41if+IRavNcKOz09FwEXDBG6ORh6iMsTSelhkMA= k8s.io/apimachinery v0.21.0/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY= -k8s.io/apiserver v0.21.0 h1:1hWMfsz+cXxB77k6/y0XxWxwl6l9OF26PC9QneUVn1Q= +k8s.io/apiserver v0.17.4/go.mod h1:5ZDQ6Xr5MNBxyi3iUZXS84QOhZl+W7Oq2us/29c0j9I= +k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= +k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= +k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= k8s.io/apiserver v0.21.0/go.mod h1:w2YSn4/WIwYuxG5zJmcqtRdtqgW/J2JRgFAqps3bBpg= +k8s.io/apiserver v0.22.1/go.mod h1:2mcM6dzSt+XndzVQJX21Gx0/Klo7Aen7i0Ai6tIa400= +k8s.io/apiserver v0.22.4 h1:L+220cy+94UWmyBl1kiVTklBXrBtKsbjlPV60eL2u6s= +k8s.io/apiserver v0.22.4/go.mod h1:38WmcUZiiy41A7Aty8/VorWRa8vDGqoUzDf2XYlku0E= k8s.io/cli-runtime v0.21.0 h1:/V2Kkxtf6x5NI2z+Sd/mIrq4FQyQ8jzZAUD6N5RnN7Y= k8s.io/cli-runtime v0.21.0/go.mod h1:XoaHP93mGPF37MkLbjGVYqg3S1MnsFdKtiA/RZzzxOo= k8s.io/client-go v0.21.0 h1:n0zzzJsAQmJngpC0IhgFcApZyoGXPrDIAD601HD09ag= k8s.io/client-go v0.21.0/go.mod h1:nNBytTF9qPFDEhoqgEPaarobC8QPae13bElIVHzIglA= +k8s.io/cloud-provider v0.17.4/go.mod h1:XEjKDzfD+b9MTLXQFlDGkk6Ho8SGMpaU8Uugx/KNK9U= +k8s.io/code-generator v0.17.2/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s= k8s.io/code-generator v0.21.0/go.mod h1:hUlps5+9QaTrKx+jiM4rmq7YmH8wPOIko64uZCHDh6Q= +k8s.io/component-base v0.17.4/go.mod h1:5BRqHMbbQPm2kKu35v3G+CpVq4K0RJKC7TRioF0I9lE= +k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= +k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= +k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= k8s.io/component-base v0.21.0/go.mod h1:qvtjz6X0USWXbgmbfXR+Agik4RZ3jv2Bgr5QnZzdPYw= -k8s.io/component-base v0.22.1 h1:SFqIXsEN3v3Kkr1bS6rstrs1wd45StJqbtgbQ4nRQdo= k8s.io/component-base v0.22.1/go.mod h1:0D+Bl8rrnsPN9v0dyYvkqFfBeAd4u7n77ze+p8CMiPo= +k8s.io/component-base v0.22.2/go.mod h1:5Br2QhI9OTe79p+TzPe9JKNQYvEKbq9rTJDWllunGug= +k8s.io/component-base v0.22.4 h1:7qwLJnua2ppGNZrRGDQ0vhsFebI39VGbZ4zdR5ArViI= +k8s.io/component-base v0.22.4/go.mod h1:MrSaQy4a3tFVViff8TZL6JHYSewNCLshZCwHYM58v5A= k8s.io/component-helpers v0.21.0/go.mod h1:tezqefP7lxfvJyR+0a+6QtVrkZ/wIkyMLK4WcQ3Cj8U= +k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= +k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= +k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= +k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc= +k8s.io/csi-translation-lib v0.17.4/go.mod h1:CsxmjwxEI0tTNMzffIAcgR9lX4wOh6AKHdxQrT7L0oo= +k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= +k8s.io/helm v2.17.0+incompatible h1:Bpn6o1wKLYqKM3+Osh8e+1/K2g/GsQJ4F4yNF2+deao= +k8s.io/helm v2.17.0+incompatible/go.mod h1:LZzlS4LQBHfciFOurYBFkCMTaZ0D1l+p0teMg7TSULI= +k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= +k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0= +k8s.io/kube-openapi v0.0.0-20180731170545-e3762e86a74c/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= +k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= +k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= +k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= +k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c h1:jvamsI1tn9V0S8jicyX82qaFC0H/NKxv2e5mbqsgR80= +k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/kubectl v0.21.0 h1:WZXlnG/yjcE4LWO2g6ULjFxtzK6H1TKzsfaBFuVIhNg= k8s.io/kubectl v0.21.0/go.mod h1:EU37NukZRXn1TpAkMUoy8Z/B2u6wjHDS4aInsDzVvks= +k8s.io/kubernetes v1.11.10/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= +k8s.io/kubernetes v1.13.0 h1:qTfB+u5M92k2fCCCVP2iuhgwwSOv1EkAkvQY1tQODD8= +k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= +k8s.io/legacy-cloud-providers v0.17.4/go.mod h1:FikRNoD64ECjkxO36gkDgJeiQWwyZTuBkhu+yxOc1Js= k8s.io/metrics v0.21.0/go.mod h1:L3Ji9EGPP1YBbfm9sPfEXSpnj8i24bfQbAFAsW0NueQ= +k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210707171843-4b05e18ac7d9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210802155522-efc7438f0176 h1:Mx0aa+SUAcNRQbs5jUzV8lkDlGFU8laZsY9jrcVX5SY= -k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a h1:8dYfu/Fc9Gz2rNJKB9IQRGgQOh2clmRzNIPPY1xLY5g= +k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= +modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= +modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= +modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= +mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc= +mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4= +mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw= +mvdan.cc/unparam v0.0.0-20200501210554-b37ab49443f7/go.mod h1:HGC5lll35J70Y5v7vCGb9oLhHoScFwkHDJm/05RdSTc= +oras.land/oras-go v0.4.0 h1:u6+7D+raZDYHwlz/uOwNANiRmyYDSSMW7A9E1xXycUQ= +oras.land/oras-go v0.4.0/go.mod h1:VJcU+VE4rkclUbum5C0O7deEZbBYnsnpbGSACwTjOcg= +pack.ag/amqp v0.11.2/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/letsencrypt v0.0.3 h1:H7xDfhkaFFSYEJlKeq38RwX2jYcnTeHuDQyT+mMNMwM= rsc.io/letsencrypt v0.0.3/go.mod h1:buyQKZ6IXrRnB7TdkHP0RyEybLx18HHyOSoTyoOLqNY= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/controller-runtime v0.10.0 h1:HgyZmMpjUOrtkaFtCnfxsR1bGRuFoAczSNbn2MoKj5U= -sigs.k8s.io/controller-runtime v0.10.0/go.mod h1:GCdh6kqV6IY4LK0JLwX0Zm6g233RtVGdb/f0+KSfprg= -sigs.k8s.io/kustomize/api v0.8.5 h1:bfCXGXDAbFbb/Jv5AhMj2BB8a5VAJuuQ5/KU69WtDjQ= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/controller-runtime v0.10.1 h1:+eLHgY/VrJWnfg6iXUqhCUqNXgPH1NZeP9drNAAgWlg= +sigs.k8s.io/controller-runtime v0.10.1/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY= sigs.k8s.io/kustomize/api v0.8.5/go.mod h1:M377apnKT5ZHJS++6H4rQoCHmWtt6qTpp3mbe7p6OLY= +sigs.k8s.io/kustomize/api v0.8.11 h1:LzQzlq6Z023b+mBtc6v72N2mSHYmN8x7ssgbf/hv0H8= +sigs.k8s.io/kustomize/api v0.8.11/go.mod h1:a77Ls36JdfCWojpUqR6m60pdGY1AYFix4AH83nJtY1g= sigs.k8s.io/kustomize/cmd/config v0.9.7/go.mod h1:MvXCpHs77cfyxRmCNUQjIqCmZyYsbn5PyQpWiq44nW0= sigs.k8s.io/kustomize/kustomize/v4 v4.0.5/go.mod h1:C7rYla7sI8EnxHE/xEhRBSHMNfcL91fx0uKmUlUhrBk= -sigs.k8s.io/kustomize/kyaml v0.10.15 h1:dSLgG78KyaxN4HylPXdK+7zB3k7sW6q3IcCmcfKA+aI= sigs.k8s.io/kustomize/kyaml v0.10.15/go.mod h1:mlQFagmkm1P+W4lZJbJ/yaxMd8PqMRSC4cPcfUVt5Hg= +sigs.k8s.io/kustomize/kyaml v0.11.0 h1:9KhiCPKaVyuPcgOLJXkvytOvjMJLoxpjodiycb4gHsA= +sigs.k8s.io/kustomize/kyaml v0.11.0/go.mod h1:GNMwjim4Ypgp/MueD3zXHLRJEjz7RvtPae0AwlvEMFM= +sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= +sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06 h1:zD2IemQ4LmOcAumeiyDWXKUI2SO0NYDe3H6QGvPOVgU= +sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06/go.mod h1:/ULNhyfzRopfcjskuui0cTITekDduZ7ycKN3oUT9R18= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8= +sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= +sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= +sourcegraph.com/sqs/pbtypes v1.0.0/go.mod h1:3AciMUv4qUuRHRHhOG4TZOB+72GdPVz5k+c648qsFS4= diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt index 65eb5b0bb3daac973649012e69dff32b727c5c53..725b5825d19d164eaeae8ccf08ab1c9a83f943ba 100644 --- a/hack/boilerplate.go.txt +++ b/hack/boilerplate.go.txt @@ -1,5 +1,5 @@ /* -Copyright 2021 The KodeRover Authors. +Copyright 2022 The KodeRover Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/hack/update-copyright.sh b/hack/update-copyright.sh index 2d57741ef64a22567c5c78f59f6836794c5f860e..1533051b917f34433645d5cda416c4a7e25dfbe3 100755 --- a/hack/update-copyright.sh +++ b/hack/update-copyright.sh @@ -1,7 +1,7 @@ #!/bin/bash -e # -# Copyright 2021 The KodeRover Authors. +# Copyright 2022 The KodeRover Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/pkg/tool/ilyshin/tag.go b/pkg/cli/initconfig/cmd/healthz.go similarity index 42% rename from pkg/tool/ilyshin/tag.go rename to pkg/cli/initconfig/cmd/healthz.go index cfbb81efd50441c13796f146de346d6b5ce6827c..bdc4d036e6c8fb56a34088aac8f1025a8dbf58b3 100644 --- a/pkg/tool/ilyshin/tag.go +++ b/pkg/cli/initconfig/cmd/healthz.go @@ -14,40 +14,38 @@ See the License for the specific language governing permissions and limitations under the License. */ -package ilyshin +package cmd import ( "fmt" - "go.uber.org/zap" - - "github.com/koderover/zadig/pkg/tool/httpclient" + "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/shared/client/aslan" + "github.com/koderover/zadig/pkg/shared/client/policy" + "github.com/koderover/zadig/pkg/shared/client/user" ) -type Tag struct { - Name string `json:"name"` - Message string `json:"message"` - Release *ReleaseNote `json:"release"` - Commit *Commit `json:"commit"` +func Healthz() error { + if err := checkUserServiceHealth(); err != nil { + return fmt.Errorf("checkUserServiceHealth error:%s", err) + } + if err := checkPolicyServiceHealth(); err != nil { + return fmt.Errorf("checkPolicyServiceHealth error:%s", err) + } + if err := checkAslanServiceHealth(); err != nil { + return fmt.Errorf("checkPolicyServiceHealth error:%s", err) + } + return nil } -type ReleaseNote struct { - TagName string `json:"tag_name"` - Description string `json:"description"` +func checkUserServiceHealth() error { + return user.New().Healthz() } -func (c *Client) ListTags(owner, repo string, log *zap.SugaredLogger) ([]*Tag, error) { - url := fmt.Sprintf("/api/v4/projects/%s/repository/tags", generateProjectName(owner, repo)) - qs := map[string]string{ - "per_page": "100", - } - - var err error - var tags []*Tag - if _, err = c.Get(url, httpclient.SetQueryParams(qs), httpclient.SetResult(&tags)); err != nil { - log.Errorf("Failed to list project tags, error: %s", err) - return tags, err - } +func checkPolicyServiceHealth() error { + return policy.NewDefault().Healthz() +} - return tags, nil +func checkAslanServiceHealth() error { + return aslan.New(config.AslanServiceAddress()).Healthz() } diff --git a/pkg/cli/initconfig/cmd/init.go b/pkg/cli/initconfig/cmd/init.go new file mode 100644 index 0000000000000000000000000000000000000000..e60ee3953ac39f368a5c8d1f22bf7c745f8cc3dd --- /dev/null +++ b/pkg/cli/initconfig/cmd/init.go @@ -0,0 +1,191 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + _ "embed" + "encoding/base64" + "encoding/json" + "time" + + "github.com/spf13/cobra" + + "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/client/aslan" + "github.com/koderover/zadig/pkg/shared/client/policy" + "github.com/koderover/zadig/pkg/shared/client/user" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" + "github.com/koderover/zadig/pkg/tool/httpclient" + "github.com/koderover/zadig/pkg/tool/kube/updater" + "github.com/koderover/zadig/pkg/tool/log" +) + +func init() { + rootCmd.AddCommand(initCmd) + log.Init(&log.Config{ + Level: config.LogLevel(), + }) +} + +var initCmd = &cobra.Command{ + Use: "init", + Short: "init system config", + Long: `init system config.`, + Run: func(cmd *cobra.Command, args []string) { + if err := run(); err != nil { + log.Fatal(err) + } + }, +} + +func run() error { + for { + err := Healthz() + if err == nil { + break + } + log.Error(err) + time.Sleep(10 * time.Second) + } + err := initSystemConfig() + if err == nil { + log.Info("zadig init success") + } + return err +} + +func initSystemConfig() error { + email := config.AdminEmail() + password := config.AdminPassword() + domain := config.SystemAddress() + + uid, err := presetSystemAdmin(email, password, domain) + if err != nil { + log.Errorf("presetSystemAdmin err:%s", err) + return err + } + + if err := presetRoleBinding(uid); err != nil { + log.Errorf("presetRoleBinding err:%s", err) + return err + } + + if err := createLocalCluster(); err != nil { + log.Errorf("createLocalCluster err:%s", err) + return err + } + + if err := scaleWarpdrive(); err != nil { + log.Errorf("scale warpdrive err: %s", err) + return err + } + + return nil +} + +func scaleWarpdrive() error { + cfg, err := aslan.New(config.AslanServiceAddress()).GetWorkflowConcurrencySetting() + if err == nil { + client, err := kubeclient.GetKubeClient(config.HubServerServiceAddress(), setting.LocalClusterID) + if err != nil { + return err + } + return updater.ScaleDeployment(config.Namespace(), config.WarpDriveServiceName(), int(cfg.WorkflowConcurrency), client) + } + + log.Errorf("Failed to get workflow concurrency settings, error: %s", err) + return err +} + +func presetSystemAdmin(email string, password, domain string) (string, error) { + r, err := user.New().SearchUser(&user.SearchUserArgs{Account: setting.PresetAccount}) + if err != nil { + log.Errorf("SearchUser err:%s", err) + return "", err + } + if len(r.Users) > 0 { + log.Infof("User admin exists, skip it.") + return r.Users[0].UID, nil + } + user, err := user.New().CreateUser(&user.CreateUserArgs{ + Name: setting.PresetAccount, + Password: password, + Account: setting.PresetAccount, + Email: email, + }) + if err != nil { + log.Errorf("created admin err:%s", err) + return "", err + } + // report register + err = reportRegister(domain, email) + if err != nil { + log.Errorf("reportRegister err: %s", err) + } + return user.Uid, nil +} + +type Operation struct { + Data string `json:"data"` +} +type Register struct { + Domain string `json:"domain"` + Username string `json:"username"` + Email string `json:"email"` + CreatedAt int64 `json:"created_at"` +} + +func reportRegister(domain, email string) error { + register := Register{ + Domain: domain, + Username: "admin", + Email: email, + CreatedAt: time.Now().Unix(), + } + registerByte, _ := json.Marshal(register) + encrypt, err := RSAEncrypt([]byte(registerByte)) + if err != nil { + log.Errorf("RSAEncrypt err: %s", err) + return err + } + encodeString := base64.StdEncoding.EncodeToString(encrypt) + reqBody := Operation{Data: encodeString} + _, err = httpclient.Post("https://api.koderover.com/api/operation/admin/user", httpclient.SetBody(reqBody)) + return err +} + +func presetRoleBinding(uid string) error { + return policy.NewDefault().CreateOrUpdateSystemRoleBinding(&policy.RoleBinding{ + Name: config.RoleBindingNameFromUIDAndRole(uid, setting.SystemAdmin, "*"), + UID: uid, + Role: string(setting.SystemAdmin), + Type: setting.ResourceTypeSystem, + }) + +} + +func createLocalCluster() error { + cluster, err := aslan.New(config.AslanServiceAddress()).GetLocalCluster() + if err != nil { + return err + } + if cluster != nil { + return nil + } + return aslan.New(config.AslanServiceAddress()).AddLocalCluster() +} diff --git a/pkg/cli/initconfig/cmd/root.go b/pkg/cli/initconfig/cmd/root.go new file mode 100644 index 0000000000000000000000000000000000000000..082fdcd40f4534ca29e7b5b242f6713efd5ef8d6 --- /dev/null +++ b/pkg/cli/initconfig/cmd/root.go @@ -0,0 +1,48 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "github.com/koderover/zadig/pkg/tool/log" +) + +var rootCmd = &cobra.Command{ + Use: "config", + Short: "init config for zadig", + Long: `init system config for zadig`, +} + +// Execute executes the root command. +func Execute() error { + return rootCmd.Execute() +} + +func init() { + cobra.OnInitialize(initConfig) +} + +func initConfig() { + viper.AutomaticEnv() + + log.Init(&log.Config{ + Level: "debug", + NoCaller: true, + }) +} diff --git a/pkg/cli/initconfig/cmd/rsa.go b/pkg/cli/initconfig/cmd/rsa.go new file mode 100644 index 0000000000000000000000000000000000000000..3e8e6bb343e8f699e08768c84e6a3fac4a856694 --- /dev/null +++ b/pkg/cli/initconfig/cmd/rsa.go @@ -0,0 +1,78 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "errors" + "os" +) + +var pub *rsa.PublicKey + +func RSAEncrypt(plainText []byte) ([]byte, error) { + if err := LoadPubKey(""); err != nil { + return nil, err + } + //对明文进行加密 + cipherText, err := rsa.EncryptPKCS1v15(rand.Reader, pub, plainText) + if err != nil { + return nil, err + } + //返回密文 + return cipherText, nil +} + +// LoadPubKey ... +func LoadPubKey(filename string) (err error) { + var block *pem.Block + if filename == "" { + block, _ = pem.Decode([]byte(defaultPublicKey)) + } else { + b, err := os.ReadFile(filename) + if err != nil { + return err + } + block, _ = pem.Decode(b) + } + if block == nil { + return errors.New("public key error") + } + pubInterface, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + return + } + pub = pubInterface.(*rsa.PublicKey) + return +} + +var defaultPublicKey = ` +-----BEGIN RSA PUBLIC KEY----- +MIIBpTANBgkqhkiG9w0BAQEFAAOCAZIAMIIBjQKCAYQAz5IqagSbovHGXmUf7wTB +XrR+DZ0u3p5jsgJW08ISJl83t0rCCGMEtcsRXJU8bE2dIIfndNwvmBiqh13/WnJd ++jgyIm6i1ZfNmf/R8pEqVXpOAOuyoD3VLT9tfWvz9nPQbjVI+PsUHH7nVR0Jwxem +NsH/7MC2O15t+2DVC1533UlhjT/pKFDdTri0mgDrLZHp6gPF5d7/yQ7cPbzv6/0p +0UgIdStT7IhkDfsJDRmLAz09znv5tQQtHfJIMdAKxwHw9mExcL2gE40sOezrgj7m +srOnJd65N8anoMGxQqNv+ycAHB9aI1Yrtgue2KKzpI/Fneghd/ZavGVFWKDYoFP3 +531Ga/CiCwtKfM0vQezfLZKAo3qpb0Edy2BcDHhLwidmaFwh8ZlXuaHbNaF/FiVR +h7uu0/9B/gn81o2f+c8GSplWB5bALhQH8tJZnvmWZGI9OnrIlWmQZsuUBooTul9Q +ZJ/w3sE1Zoxa+Or1/eWijqtIfhukOJBNyGaj+esFg6uEeBgHAgMBAAE= +-----END RSA PUBLIC KEY----- +` diff --git a/pkg/cli/upgradeassistant/cmd/migrate.go b/pkg/cli/upgradeassistant/cmd/migrate.go index f02b2863141aca75654d95c1a335bb2a411ee2d4..5d69808672757c665a767eac95e2cc80185eeaab 100644 --- a/pkg/cli/upgradeassistant/cmd/migrate.go +++ b/pkg/cli/upgradeassistant/cmd/migrate.go @@ -21,15 +21,16 @@ import ( "fmt" "time" + "github.com/blang/semver/v4" "github.com/spf13/cobra" "github.com/spf13/viper" + "k8s.io/apimachinery/pkg/util/sets" _ "github.com/koderover/zadig/pkg/cli/upgradeassistant/cmd/migrate" "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/upgradepath" "github.com/koderover/zadig/pkg/setting" "github.com/koderover/zadig/pkg/tool/log" mongotool "github.com/koderover/zadig/pkg/tool/mongo" - "github.com/koderover/zadig/version" ) const oldestVersion = "1.3.0" @@ -38,7 +39,7 @@ func init() { rootCmd.AddCommand(migrateCmd) migrateCmd.PersistentFlags().StringP("from-version", "f", oldestVersion, "current version to migrate from") - migrateCmd.PersistentFlags().StringP("to-version", "t", version.Version, "target version to migrate to") + migrateCmd.PersistentFlags().StringP("to-version", "t", "", "target version to migrate to") _ = viper.BindPFlag("fromVersion", migrateCmd.PersistentFlags().Lookup("from-version")) _ = viper.BindPFlag("toVersion", migrateCmd.PersistentFlags().Lookup("to-version")) } @@ -63,7 +64,47 @@ var migrateCmd = &cobra.Command{ } func run() error { - err := upgradepath.UpgradeWithBestPath(viper.GetString("fromVersion"), viper.GetString("toVersion")) + from := viper.GetString("fromVersion") + to := viper.GetString("toVersion") + + log.Infof("Migrating from %s to %s", from, to) + + versionSets := sets.NewString() + for _, rh := range upgradepath.RegisteredHandlers { + versionSets.Insert(rh.FromVersion, rh.ToVersion) + } + + if len(from) == 0 { + from = oldestVersion + } + if len(to) == 0 { + return fmt.Errorf("target version not assigned") + } + versionSets.Insert(from, to) + + for _, verStr := range versionSets.List() { + semVersion, err := semver.Make(verStr) + if err != nil { + return fmt.Errorf("failed to parse version: %s, err: %s", verStr, err) + } + upgradepath.VersionDatas = append(upgradepath.VersionDatas, semVersion) + } + + // sort versions by semver orders + semver.Sort(upgradepath.VersionDatas) + + // add default handlers + for i := 0; i < len(upgradepath.VersionDatas)-1; i++ { + upgradepath.AddHandler(i, i+1, upgradepath.DefaultUpgradeHandler) + upgradepath.AddHandler(i+1, i, upgradepath.DefaultRollBackHandler) + } + + // add custom handlers + for _, rh := range upgradepath.RegisteredHandlers { + upgradepath.AddHandler(upgradepath.VersionDatas.VersionIndex(rh.FromVersion), upgradepath.VersionDatas.VersionIndex(rh.ToVersion), rh.Fn) + } + + err := upgradepath.UpgradeWithBestPath(from, to) if err == nil { log.Info("Migration finished") } diff --git a/pkg/cli/upgradeassistant/cmd/migrate/1100.go b/pkg/cli/upgradeassistant/cmd/migrate/1100.go new file mode 100644 index 0000000000000000000000000000000000000000..443a2d3f083db60d739918b0736f1cebffd41c2d --- /dev/null +++ b/pkg/cli/upgradeassistant/cmd/migrate/1100.go @@ -0,0 +1,426 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrate + +import ( + "context" + "fmt" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/repository/models" + internalmodels "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/repository/models" + internalmongodb "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/repository/mongodb" + "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/repository/orm" + "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/upgradepath" + "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/log" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" + "github.com/koderover/zadig/pkg/types" +) + +func init() { + upgradepath.RegisterHandler("1.9.0", "1.10.0", V190ToV1100) + upgradepath.RegisterHandler("1.10.0", "1.9.0", V1100ToV190) +} + +// V190ToV1100 migrates data `caches` and `pre_build.clean_workspace` fields in `zadig.module_build` and `zadig.module_testing` +// to new fields `cache_enable`, `cache_dir_type` and `cache_user_dir`; generate labelBindings for production environment +func V190ToV1100() error { + log.Info("Migrating data from 1.9.0 to 1.10.0") + + if err := changePolicyCollectionName(); err != nil { + log.Errorf("Failed to changePolicyCollectionName, err: %s", err) + return err + } + + if err := addPresetRoleSystemType(); err != nil { + log.Errorf("Failed to addPresetRoleSystemType, err: %s", err) + return err + } + + if err := addRoleBindingSystemType(); err != nil { + log.Errorf("Failed to addRoleBindingSystemType, err: %s", err) + return err + } + + if err := migrateModuleBuild(); err != nil { + return fmt.Errorf("failed to migrate data in `zadig.module_build`: %s", err) + } + + log.Info("Migrate data in `zadig.module_testing`.") + if err := migrateModuleTesting(); err != nil { + return fmt.Errorf("failed to migrate data in `zadig.module_testing`: %s", err) + } + + log.Info("UpdateUserDBTables: ADD cloumn from mysql table `user`.") + if err := orm.UpdateUserDBTables(orm.DbEditActionAdd); err != nil { + return fmt.Errorf("UpdateUserDBTables: failed to ADD cloumn from mysql table `user`: %s", err) + } + return nil +} + +func changeToCustomType() error { + ctx := context.Background() + + var res []*models.RoleBinding + cursor, err := newRoleBindingColl().Find(ctx, bson.M{}) + if err != nil { + log.Errorf("Failed to Find Policies, err: %s", err) + return err + } + + err = cursor.All(ctx, &res) + if err != nil { + return err + } + for _, v := range res { + if v.Namespace == "*" && v.RoleRef.Name == "admin" && v.Type != "" { + continue + } + query := bson.M{"name": v.Name} + change := bson.M{"$set": bson.M{ + "type": setting.ResourceTypeCustom, + }} + _, err := newRoleBindingColl().UpdateOne(ctx, query, change) + if err != nil { + return err + } + + } + return nil +} + +func changeToSystemType() error { + query := bson.M{"namespace": "*", "role_ref.name": "admin"} + change := bson.M{"$set": bson.M{ + "type": setting.ResourceTypeSystem, + }} + _, err := newRoleBindingColl().UpdateOne(context.TODO(), query, change) + if err != nil { + return err + } + return nil +} + +func addRoleBindingSystemType() error { + if err := changeToSystemType(); err != nil { + return err + } + return changeToCustomType() +} + +func addPresetRoleSystemType() error { + ctx := context.Background() + + var res []*models.Role + cursor, err := newRoleColl().Find(ctx, bson.M{}) + if err != nil { + log.Errorf("Failed to Find Policies, err: %s", err) + return err + } + + err = cursor.All(ctx, &res) + if err != nil { + return err + } + for _, v := range res { + if v.Namespace == "*" || v.Namespace == "" { + query := bson.M{"name": v.Name} + change := bson.M{"$set": bson.M{ + "type": setting.ResourceTypeSystem, + }} + _, err := newRoleColl().UpdateOne(ctx, query, change) + if err != nil { + return err + } + } else { + query := bson.M{"name": v.Name} + change := bson.M{"$set": bson.M{ + "type": setting.ResourceTypeCustom, + }} + _, err := newRoleColl().UpdateOne(ctx, query, change) + if err != nil { + return err + } + } + } + return nil +} + +func changePolicyCollectionName() error { + var res []*models.PolicyMeta + + ctx := context.Background() + cursor, err := newPolicyColl().Find(ctx, bson.M{}) + if err != nil { + log.Errorf("Failed to Find Policies, err: %s", err) + return err + } + + err = cursor.All(ctx, &res) + if err != nil { + return err + } + + for _, v := range res { + query := bson.M{"resource": v.Resource} + opts := options.Replace().SetUpsert(true) + _, err := newPolicyMetaColl().ReplaceOne(context.TODO(), query, v, opts) + if err != nil { + log.Errorf("relace one err:%v ,resource:%v", err, v.Resource) + } + } + //delete collection + return newPolicyColl().Drop(ctx) +} + +func rollbackChangePolicyCollectionName() error { + var res []*models.PolicyMeta + ctx := context.Background() + cursor, err := newPolicyMetaColl().Find(ctx, bson.M{}) + if err != nil { + return err + } + + err = cursor.All(ctx, &res) + if err != nil { + return err + } + + var ois []interface{} + for _, obj := range res { + ois = append(ois, obj) + } + if _, err = newPolicyColl().InsertMany(ctx, ois); err != nil { + log.Errorf("Failed to InsertMany, err: %s", err) + return err + } + //delete collection + return newPolicyMetaColl().Drop(ctx) +} + +func V1100ToV190() error { + log.Info("Rollback data from 1.10.0 to 1.9.0") + if err := rollbackChangePolicyCollectionName(); err != nil { + log.Errorf("Failed to rollbackChangePolicyCollectionName,err: %s", err) + return err + } + + log.Info("UpdateUserDBTables: drop cloumn from mysql table `user`.") + if err := orm.UpdateUserDBTables(orm.DbEditActionDrop); err != nil { + return fmt.Errorf("UpdateUserDBTables: failed to drop cloumn from mysql table `user`: %s", err) + } + return nil +} + +func newPolicyColl() *mongo.Collection { + collection := mongotool.Database(fmt.Sprintf("%s_policy", config.MongoDatabase())).Collection("policy") + return collection +} + +func newRoleColl() *mongo.Collection { + collection := mongotool.Database(fmt.Sprintf("%s_policy", config.MongoDatabase())).Collection("role") + return collection +} + +func newPolicyMetaColl() *mongo.Collection { + collection := mongotool.Database(fmt.Sprintf("%s_policy", config.MongoDatabase())).Collection("policy_meta") + return collection +} + +func migrateModuleBuild() error { + buildCol := internalmongodb.NewBuildColl() + + builds, err := buildCol.List(&internalmongodb.BuildListOption{}) + if err != nil { + return fmt.Errorf("failed to list all data in `zadig.module_build`: %s", err) + } + + if len(builds) == 0 { + return nil + } + + var ms []mongo.WriteModel + for _, build := range builds { + if err := migrateOneBuild(build); err != nil { + log.Errorf("Failed to migrate data: %v. Err: %s", build, err) + continue + } + + ms = append(ms, + mongo.NewUpdateOneModel(). + SetFilter(bson.D{{"_id", build.ID}}). + SetUpdate(bson.D{{"$set", + bson.D{ + {"cache_enable", build.CacheEnable}, + {"cache_dir_type", build.CacheDirType}, + {"cache_user_dir", build.CacheUserDir}, + {"advanced_setting_modified", build.AdvancedSettingsModified}, + }}, + }), + ) + } + + _, err = buildCol.BulkWrite(context.TODO(), ms) + + return err +} + +func migrateModuleTesting() error { + testingCol := internalmongodb.NewTestingColl() + + testings, err := testingCol.List(&internalmongodb.ListTestOption{}) + if err != nil { + return fmt.Errorf("failed to list all data in `zadig.module_testing`: %s", err) + } + + if len(testings) == 0 { + return nil + } + + var ms []mongo.WriteModel + for _, testing := range testings { + if err := migrateOneTesting(testing); err != nil { + log.Errorf("Failed to migrate data: %v. Err: %s", testing, err) + continue + } + + ms = append(ms, + mongo.NewUpdateOneModel(). + SetFilter(bson.D{{"_id", testing.ID}}). + SetUpdate(bson.D{{"$set", + bson.D{ + {"cache_enable", testing.CacheEnable}, + {"cache_dir_type", testing.CacheDirType}, + {"cache_user_dir", testing.CacheUserDir}, + {"advanced_setting_modified", testing.AdvancedSettingsModified}, + }}, + }), + ) + } + + _, err = testingCol.BulkWrite(context.TODO(), ms) + + return err +} + +func migrateOneBuild(build *internalmodels.Build) error { + if build.PreBuild != nil && build.PreBuild.CleanWorkspace { + return nil + } + + build.CacheEnable = true + + if len(build.Caches) == 0 { + build.CacheDirType = types.WorkspaceCacheDir + return nil + } + + build.CacheDirType = types.UserDefinedCacheDir + build.CacheUserDir = build.Caches[0] + + build.AdvancedSettingsModified = buildHasModifiedAdvancedSetting(build) + + return nil +} + +func migrateOneTesting(testing *internalmodels.Testing) error { + if testing.PreTest != nil && testing.PreTest.CleanWorkspace { + return nil + } + + testing.CacheEnable = true + + if len(testing.Caches) == 0 { + testing.CacheDirType = types.WorkspaceCacheDir + return nil + } + + testing.CacheDirType = types.UserDefinedCacheDir + testing.CacheUserDir = testing.Caches[0] + + testing.AdvancedSettingsModified = testHasModifiedAdvancedSetting(testing) + + return nil +} + +func buildHasModifiedAdvancedSetting(build *internalmodels.Build) bool { + // default timeout is 60 + if build.Timeout != 60 { + return true + } + + // cache is disabled by default + if build.CacheEnable { + return true + } + + // default is local cluster, but it is a relatively new field, so we also check for empty + if build.PreBuild.ClusterID != "" && build.PreBuild.ClusterID != setting.LocalClusterID { + return true + } + + if build.PreBuild.ResReq != setting.LowRequest { + return true + } + + return false +} + +func testHasModifiedAdvancedSetting(testing *internalmodels.Testing) bool { + // default no artifact path exists + if testing.ArtifactPaths != nil && len(testing.ArtifactPaths) > 0 { + return true + } + + // default timeout is 60 + if testing.Timeout != 60 { + return true + } + + // cache is disabled by default + if testing.CacheEnable { + return true + } + + // default is local cluster, but it is a relatively new field, so we also check for empty + if testing.PreTest.ClusterID != "" && testing.PreTest.ClusterID != setting.LocalClusterID { + return true + } + + if testing.PreTest.ResReq != setting.LowRequest { + return true + } + + if testing.HookCtl != nil && testing.HookCtl.Enabled { + return true + } + + if testing.NotifyCtl != nil && testing.NotifyCtl.Enabled { + return true + } + + if testing.Schedules != nil && testing.Schedules.Enabled { + return true + } + + return false +} diff --git a/pkg/cli/upgradeassistant/cmd/migrate/1110.go b/pkg/cli/upgradeassistant/cmd/migrate/1110.go new file mode 100644 index 0000000000000000000000000000000000000000..cc46bd0a672eb1dd89890f4a633bf500345b3807 --- /dev/null +++ b/pkg/cli/upgradeassistant/cmd/migrate/1110.go @@ -0,0 +1,133 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrate + +import ( + "context" + + "go.mongodb.org/mongo-driver/bson" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/repository/models" + "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/upgradepath" + "github.com/koderover/zadig/pkg/tool/log" +) + +func init() { + upgradepath.RegisterHandler("1.10.0", "1.11.0", V1100ToV1110) + upgradepath.RegisterHandler("1.11.0", "1.10.0", V1110ToV1100) +} + +func V1100ToV1110() error { + // iterate all roles , if the role contains manage env , add debug_pod + if err := roleAddPodDebug(); err != nil { + log.Errorf("roleAddPodDebug err:%s", err) + return err + } + + return nil +} + +func V1110ToV1100() error { + if err := roleDeletePodDebug(); err != nil { + log.Errorf("roleDeletePodDebug err:%s", err) + return err + } + return nil +} + +func roleAddPodDebug() error { + log.Info("start to scan role") + ctx := context.Background() + var res []*models.Role + cursor, err := newRoleColl().Find(ctx, bson.M{}) + if err != nil { + log.Errorf("fail to find roles, err: %s", err) + return err + } + err = cursor.All(ctx, &res) + if err != nil { + log.Errorf("get roles err:%s", err) + return err + } + for _, role := range res { + log.Infof("start to check role:%s", role.Name) + toUpdate := false + for i, rule := range role.Rules { + verbsSet := sets.NewString(rule.Verbs...) + if verbsSet.Has("manage_environment") { + verbsSet.Insert("debug_pod") + role.Rules[i].Verbs = verbsSet.List() + toUpdate = true + } + } + if toUpdate { + // start to update + log.Infof("find role:%s has verb:manage_environment ,start to update the role", role.Name) + query := bson.M{"name": role.Name} + change := bson.M{"$set": bson.M{ + "rules": role.Rules, + }} + _, err := newRoleColl().UpdateOne(ctx, query, change) + if err != nil { + log.Errorf("fail to update,err:%s", err) + return err + } + } + } + return nil +} + +func roleDeletePodDebug() error { + ctx := context.Background() + var res []*models.Role + cursor, err := newRoleColl().Find(ctx, bson.M{}) + if err != nil { + log.Errorf("fail to find roles, err: %s", err) + return err + } + err = cursor.All(ctx, &res) + if err != nil { + log.Errorf("get roles err:%s", err) + return err + } + for _, role := range res { + toUpdate := false + for i, rule := range role.Rules { + verbsSet := sets.NewString(rule.Verbs...) + if verbsSet.Has("debug_pod") { + verbsSet.Delete("debug_pod") + role.Rules[i].Verbs = verbsSet.List() + toUpdate = true + } + } + if toUpdate { + // start to update + log.Infof("find role:%s has verb:debug_pod ,start to update the role", role.Name) + query := bson.M{"name": role.Name} + change := bson.M{"$set": bson.M{ + "rules": role.Rules, + }} + _, err := newRoleColl().UpdateOne(ctx, query, change) + if err != nil { + log.Errorf("err update%s\n", err) + return err + } + } + } + return nil +} diff --git a/pkg/cli/upgradeassistant/cmd/migrate/1120.go b/pkg/cli/upgradeassistant/cmd/migrate/1120.go new file mode 100644 index 0000000000000000000000000000000000000000..778bf20f73f0f28d9d53ef97f38945d3735ec467 --- /dev/null +++ b/pkg/cli/upgradeassistant/cmd/migrate/1120.go @@ -0,0 +1,408 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrate + +import ( + "context" + "fmt" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + + "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/repository/models" + internalmongodb "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/repository/mongodb" + "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/upgradepath" + "github.com/koderover/zadig/pkg/tool/log" +) + +func init() { + upgradepath.RegisterHandler("1.11.0", "1.12.0", V1110ToV1120) + upgradepath.RegisterHandler("1.12.0", "1.11.0", V1120ToV1110) +} + +func V1110ToV1120() error { + // iterate all testings , if contains notifyctl , add notifyctls + if err := moduleTestingAddNotifyCtls(); err != nil { + log.Errorf("moduleTestingAddNotifyCtls err:%s", err) + return err + } + // iterate all workflows , if contains notifyctl , add notifyctls + if err := workflowAddNotifyCtls(); err != nil { + log.Errorf("workflowAddNotifyCtls err:%s", err) + return err + } + + if err := buildAddJenkinsID(); err != nil { + log.Errorf("buildAddJenkinsID err:%s", err) + return err + } + + if err := integrateSonar(); err != nil { + log.Errorf("add sonar integration err:%s", err) + return err + } + + if err := supplementCodehostAlias(); err != nil { + log.Errorf("supplement codehost alias, err:%s", err) + return err + } + + return nil +} + +func V1120ToV1110() error { + // iterate all testings , if contains notifyctl , add notifyctls + if err := moduleTestingRollBackNotifyCtls(); err != nil { + log.Errorf("moduleTestingRollBackNotifyCtls err:%s", err) + return err + } + // iterate all workflows , if contains notifyctl , add notifyctls + if err := workflowRollBackNotifyCtls(); err != nil { + log.Errorf("workflowRollBackNotifyCtls err:%s", err) + return err + } + + if err := buildAddJenkinsIDRollBack(); err != nil { + log.Errorf("buildAddJenkinsIDRollBack err:%s", err) + return err + } + + if err := supplementCodehostAliasRollBack(); err != nil { + log.Errorf("supplement codehost alias rollback, err:%s", err) + return err + } + return nil +} + +func moduleTestingAddNotifyCtls() error { + testingCol := internalmongodb.NewTestingColl() + + testings, err := testingCol.List(&internalmongodb.ListTestOption{}) + if err != nil { + return fmt.Errorf("failed to list all data in `zadig.module_testing`,err: %s", err) + } + + if len(testings) == 0 { + return nil + } + + var ms []mongo.WriteModel + for _, testing := range testings { + // 如果已经执行过数据迁移，不要重复迁移 + if testing.NotifyCtls != nil { + continue + } + if testing.NotifyCtl != nil && testing.NotifyCtl.Enabled { + testing.NotifyCtls = []*models.NotifyCtl{testing.NotifyCtl} + } + + ms = append(ms, + mongo.NewUpdateOneModel(). + SetFilter(bson.D{{"_id", testing.ID}}). + SetUpdate(bson.D{{"$set", + bson.D{ + {"notify_ctls", testing.NotifyCtls}, + }}, + }), + ) + } + if len(ms) > 0 { + _, err = testingCol.BulkWrite(context.TODO(), ms) + } + + return err +} + +func moduleTestingRollBackNotifyCtls() error { + testingCol := internalmongodb.NewTestingColl() + + testings, err := testingCol.List(&internalmongodb.ListTestOption{}) + if err != nil { + return fmt.Errorf("failed to list all data in `zadig.module_testing`,err: %s", err) + } + + if len(testings) == 0 { + return nil + } + + var ms []mongo.WriteModel + for _, testing := range testings { + for _, notifyctl := range testing.NotifyCtls { + if notifyctl != nil && notifyctl.Enabled { + testing.NotifyCtl = notifyctl + break + } + } + + ms = append(ms, + mongo.NewUpdateOneModel(). + SetFilter(bson.D{{"_id", testing.ID}}). + SetUpdate(bson.D{{"$set", + bson.D{ + {"notify_ctl", testing.NotifyCtl}, + }}, + }), + ) + } + if len(ms) > 0 { + _, err = testingCol.BulkWrite(context.TODO(), ms) + } + + return err +} + +func workflowAddNotifyCtls() error { + workflowColl := internalmongodb.NewWorkflowColl() + + workflows, err := workflowColl.List(&internalmongodb.ListWorkflowOption{}) + if err != nil { + return fmt.Errorf("failed to list all data in `zadig.workflow`,err: %s", err) + } + + if len(workflows) == 0 { + return nil + } + + var ms []mongo.WriteModel + for _, workflow := range workflows { + // 如果已经执行过数据迁移，不要重复迁移 + if workflow.NotifyCtls != nil { + continue + } + if workflow.NotifyCtl != nil && workflow.NotifyCtl.Enabled { + workflow.NotifyCtls = []*models.NotifyCtl{workflow.NotifyCtl} + } + + ms = append(ms, + mongo.NewUpdateOneModel(). + SetFilter(bson.D{{"_id", workflow.ID}}). + SetUpdate(bson.D{{"$set", + bson.D{ + {"notify_ctls", workflow.NotifyCtls}, + }}, + }), + ) + } + if len(ms) > 0 { + _, err = workflowColl.BulkWrite(context.TODO(), ms) + } + + return err +} + +func workflowRollBackNotifyCtls() error { + workflowColl := internalmongodb.NewWorkflowColl() + + workflows, err := workflowColl.List(&internalmongodb.ListWorkflowOption{}) + if err != nil { + return fmt.Errorf("failed to list all data in `zadig.workflow`,err: %s", err) + } + + if len(workflows) == 0 { + return nil + } + + var ms []mongo.WriteModel + for _, workflow := range workflows { + for _, notifyctl := range workflow.NotifyCtls { + if notifyctl != nil && notifyctl.Enabled { + workflow.NotifyCtl = notifyctl + break + } + } + + ms = append(ms, + mongo.NewUpdateOneModel(). + SetFilter(bson.D{{"_id", workflow.ID}}). + SetUpdate(bson.D{{"$set", + bson.D{ + {"notify_ctl", workflow.NotifyCtl}, + }}, + }), + ) + } + if len(ms) > 0 { + _, err = workflowColl.BulkWrite(context.TODO(), ms) + } + + return err +} + +// If the data is not cleaned, executing the jenkins build will report an error +func buildAddJenkinsID() error { + jenkins, err := internalmongodb.NewJenkinsIntegrationColl().List() + if err != nil { + return fmt.Errorf("failed to list jenkins integration ,err: %s", err) + } + + if len(jenkins) == 0 { + return nil + } + + builds, err := internalmongodb.NewBuildColl().List(&internalmongodb.BuildListOption{}) + if err != nil { + return fmt.Errorf("failed to list all builds ,err: %s", err) + } + + if len(builds) == 0 { + return nil + } + + var ms []mongo.WriteModel + for _, build := range builds { + if build.JenkinsBuild == nil { + continue + } + + if build.JenkinsBuild.JenkinsID != "" { + continue + } + + ms = append(ms, + mongo.NewUpdateOneModel(). + SetFilter(bson.D{{"_id", build.ID}}). + SetUpdate(bson.D{{"$set", + bson.D{ + {"jenkins_build.jenkins_id", jenkins[0].ID.Hex()}, + }}, + }), + ) + } + if len(ms) > 0 { + _, err = internalmongodb.NewBuildColl().BulkWrite(context.TODO(), ms) + } + + return err +} + +func integrateSonar() error { + sonarImageList, err := internalmongodb.NewBasicImageColl().GetSonarTypeImage() + if err != nil { + return fmt.Errorf("failed to list basic images of type sonar ,err: %s", err) + } + + // if no sonar image list is included, we add one + if len(sonarImageList) == 0 { + return internalmongodb.NewBasicImageColl().CreateZadigSonarImage() + } + return nil +} + +func buildAddJenkinsIDRollBack() error { + builds, err := internalmongodb.NewBuildColl().List(&internalmongodb.BuildListOption{}) + if err != nil { + return fmt.Errorf("failed to list all builds ,err: %s", err) + } + + if len(builds) == 0 { + return nil + } + + var ms []mongo.WriteModel + for _, build := range builds { + if build.JenkinsBuild == nil { + continue + } + + if build.JenkinsBuild.JenkinsID == "" { + continue + } + + ms = append(ms, + mongo.NewUpdateOneModel(). + SetFilter(bson.D{{"_id", build.ID}}). + SetUpdate(bson.D{{"$set", + bson.D{ + {"jenkins_build.jenkins_id", ""}, + }}, + }), + ) + } + if len(ms) > 0 { + _, err = internalmongodb.NewBuildColl().BulkWrite(context.TODO(), ms) + } + return err + +} +func supplementCodehostAlias() error { + codeHosts, err := internalmongodb.NewCodehostColl().List() + if err != nil { + return fmt.Errorf("failed to list all codehosts ,err: %s", err) + } + var ms []mongo.WriteModel + for _, codehost := range codeHosts { + if codehost.Alias != "" { + continue + } + var alias string + switch codehost.Type { + case "gitlab": + alias = codehost.AccessKey[0:8] + case "github": + alias = codehost.Namespace + case "gerrit": + alias = codehost.Username + case "gitee": + alias = codehost.Namespace + } + ms = append(ms, + mongo.NewUpdateOneModel(). + SetFilter(bson.D{{"id", codehost.ID}}). + SetUpdate(bson.D{{"$set", + bson.D{ + {"alias", alias}, + }}, + }), + ) + } + + if len(ms) > 0 { + _, err = internalmongodb.NewCodehostColl().BulkWrite(context.TODO(), ms) + } + return err +} + +func supplementCodehostAliasRollBack() error { + codeHosts, err := internalmongodb.NewCodehostColl().List() + if err != nil { + return fmt.Errorf("failed to list all codehosts ,err: %s", err) + } + var ms []mongo.WriteModel + for _, codehost := range codeHosts { + if codehost.Alias == "" { + continue + } + if codehost.Type == "other" { + continue + } + + ms = append(ms, + mongo.NewUpdateOneModel(). + SetFilter(bson.D{{"id", codehost.ID}}). + SetUpdate(bson.D{{"$set", + bson.D{ + {"alias", ""}, + }}, + }), + ) + } + + if len(ms) > 0 { + _, err = internalmongodb.NewCodehostColl().BulkWrite(context.TODO(), ms) + } + return err +} diff --git a/pkg/cli/upgradeassistant/cmd/migrate/1130.go b/pkg/cli/upgradeassistant/cmd/migrate/1130.go new file mode 100644 index 0000000000000000000000000000000000000000..f6633b05fa7d071ef7ff551fa114cb66a0ce8d8c --- /dev/null +++ b/pkg/cli/upgradeassistant/cmd/migrate/1130.go @@ -0,0 +1,155 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrate + +import ( + "context" + + "go.mongodb.org/mongo-driver/bson/primitive" + + "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/repository/models" + internalmongodb "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/repository/mongodb" + "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/upgradepath" + "github.com/koderover/zadig/pkg/tool/log" +) + +func init() { + upgradepath.RegisterHandler("1.12.0", "1.13.0", V1120ToV1130) + upgradepath.RegisterHandler("1.13.0", "1.12.0", V1130ToV1120) +} + +func V1120ToV1130() error { + return CombineEnvResources() +} + +func V1130ToV1120() error { + return RemoveEnvResource() +} + +func CombineEnvResources() error { + log.Info("start CombineEnvResources") + resourceList := make([]*models.EnvResource, 0) + + cms, err := internalmongodb.NewConfigMapColl().List() + if err != nil { + return err + } + resourceList = append(resourceList, ConvertCmToResource(cms)...) + + ings, err := internalmongodb.NewIngressColl().List() + if err != nil { + return err + } + resourceList = append(resourceList, ConvertIngToResource(ings)...) + + secrets, err := internalmongodb.NewSecretColl().List() + if err != nil { + return err + } + resourceList = append(resourceList, ConvertSecretToResource(secrets)...) + + pvcs, err := internalmongodb.NewPvcColl().List() + if err != nil { + return err + } + resourceList = append(resourceList, ConvertPVCToResource(pvcs)...) + + return internalmongodb.NewEnvResourceColl().BatchInsert(resourceList) +} + +func ConvertCmToResource(cms []*models.EnvConfigMap) []*models.EnvResource { + resourceList := make([]*models.EnvResource, 0, len(cms)) + for _, cm := range cms { + resourceList = append(resourceList, &models.EnvResource{ + ID: primitive.ObjectID{}, + Type: "ConfigMap", + ProductName: cm.ProductName, + CreateTime: cm.CreateTime, + UpdateUserName: cm.UpdateUserName, + Namespace: cm.Namespace, + EnvName: cm.EnvName, + Name: cm.Name, + YamlData: cm.YamlData, + DeletedAt: 0, + }) + } + return resourceList +} + +func ConvertIngToResource(ings []*models.EnvIngress) []*models.EnvResource { + resourceList := make([]*models.EnvResource, 0, len(ings)) + for _, ingress := range ings { + resourceList = append(resourceList, &models.EnvResource{ + ID: primitive.ObjectID{}, + Type: "Ingress", + ProductName: ingress.ProductName, + CreateTime: ingress.CreateTime, + UpdateUserName: ingress.UpdateUserName, + Namespace: ingress.Namespace, + EnvName: ingress.EnvName, + Status: ingress.Status, + Name: ingress.Name, + YamlData: ingress.YamlData, + DeletedAt: 0, + }) + } + return resourceList +} + +func ConvertSecretToResource(secrets []*models.EnvSecret) []*models.EnvResource { + resourceList := make([]*models.EnvResource, 0, len(secrets)) + for _, secret := range secrets { + resourceList = append(resourceList, &models.EnvResource{ + ID: primitive.ObjectID{}, + Type: "Secret", + ProductName: secret.ProductName, + CreateTime: secret.CreateTime, + UpdateUserName: secret.UpdateUserName, + Namespace: secret.Namespace, + EnvName: secret.EnvName, + Status: secret.Status, + Name: secret.Name, + YamlData: secret.YamlData, + DeletedAt: 0, + }) + } + return resourceList +} + +func ConvertPVCToResource(pvcs []*models.EnvPvc) []*models.EnvResource { + resourceList := make([]*models.EnvResource, 0, len(pvcs)) + for _, pvc := range pvcs { + resourceList = append(resourceList, &models.EnvResource{ + ID: primitive.ObjectID{}, + Type: "PVC", + ProductName: pvc.ProductName, + CreateTime: pvc.CreateTime, + UpdateUserName: pvc.UpdateUserName, + Namespace: pvc.Namespace, + EnvName: pvc.EnvName, + Status: pvc.Status, + Name: pvc.Name, + YamlData: pvc.YamlData, + DeletedAt: 0, + }) + } + return resourceList +} + +func RemoveEnvResource() error { + return internalmongodb.NewEnvResourceColl().Drop(context.TODO()) +} diff --git a/pkg/cli/upgradeassistant/cmd/migrate/150.go b/pkg/cli/upgradeassistant/cmd/migrate/150.go index 4d23841ac388ac8d8ca51dcab991e9fc68530cff..a2bba446b026bea9a4a99f365dac3f42372ad81c 100644 --- a/pkg/cli/upgradeassistant/cmd/migrate/150.go +++ b/pkg/cli/upgradeassistant/cmd/migrate/150.go @@ -20,9 +20,7 @@ import ( "fmt" "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/upgradepath" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/setting" @@ -31,8 +29,8 @@ import ( ) func init() { - upgradepath.AddHandler(upgradepath.V140, upgradepath.V150, V140ToV150) - upgradepath.AddHandler(upgradepath.V150, upgradepath.V140, V150ToV140) + upgradepath.RegisterHandler("1.4.0", "1.5.0", V140ToV150) + upgradepath.RegisterHandler("1.5.0", "1.4.0", V150ToV140) } // V140ToV150 fill image path data for old data in product.services.containers @@ -40,8 +38,8 @@ func init() { func V140ToV150() error { log.Info("Migrating data from 1.4.0 to 1.5.0") - products, err := mongodb.NewProductColl().List(&mongodb.ProductListOptions{ - ExcludeStatus: setting.ProductStatusDeleting, + products, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{ + ExcludeStatus: []string{setting.ProductStatusDeleting, setting.ProductStatusUnknown}, Source: setting.SourceFromHelm, }) if err != nil { @@ -134,7 +132,7 @@ func fillImageParseInfo(prod *commonmodels.Product) error { } // find match rule -func findImageByContainerName(flatMap map[string]interface{}, matchedPath []map[string]string, container *models.Container) error { +func findImageByContainerName(flatMap map[string]interface{}, matchedPath []map[string]string, container *commonmodels.Container) error { for _, searchResult := range matchedPath { imageURI, err := commonservice.GeneImageURI(searchResult, flatMap) if err != nil { @@ -144,12 +142,12 @@ func findImageByContainerName(flatMap map[string]interface{}, matchedPath []map[ if container.Name != commonservice.ExtractImageName(imageURI) { continue } - container.ImagePath = &models.ImagePathSpec{ + container.ImagePath = &commonmodels.ImagePathSpec{ Repo: searchResult[setting.PathSearchComponentRepo], Image: searchResult[setting.PathSearchComponentImage], Tag: searchResult[setting.PathSearchComponentTag], } return nil } - return fmt.Errorf("faild to find image for contianer %s", container.Image) + return fmt.Errorf("failed to find image for contianer %s", container.Image) } diff --git a/pkg/cli/upgradeassistant/cmd/migrate/170.go b/pkg/cli/upgradeassistant/cmd/migrate/170.go new file mode 100644 index 0000000000000000000000000000000000000000..ee721692a3b39a58ef5bd3cc653ba7a35288e362 --- /dev/null +++ b/pkg/cli/upgradeassistant/cmd/migrate/170.go @@ -0,0 +1,241 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrate + +import ( + internalmodels "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/repository/models" + internalmongodb "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/repository/mongodb" + "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/upgradepath" + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + gitservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/git" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/github" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/gitlab" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/log" +) + +func init() { + upgradepath.RegisterHandler("1.6.0", "1.7.0", V160ToV170) + upgradepath.RegisterHandler("1.7.0", "1.6.0", V170ToV160) +} + +// V160ToV170 refreshes the secret of all webhooks +func V160ToV170() error { + log.Info("Migrating data from 1.6.0 to 1.7.0") + log.Info("Start to change codeHost type") + err := changeCodehostType() + if err != nil { + log.Errorf("Failed to change codehost type, err: %s", err) + return err + } + log.Info("Start to refresh all webhook secrets") + err = refreshWebHookSecret(gitservice.GetHookSecret()) + if err != nil { + log.Errorf("Failed to refresh webhook secret, err: %s", err) + return err + } + + err = changeEmailHostType() + if err != nil { + log.Errorf("Failed to change emailhost type , err: %s", err) + return err + } + + return nil +} + +func V170ToV160() error { + log.Info("Rollback data from 1.7.0 to 1.6.0") + err := rollbackEmailhostType() + if err != nil { + log.Errorf("Failed to rollback emailhost type, err: %s", err) + return err + } + + token := getWebHookTokenFromOrganization() + if token == "" { + return nil + } + + log.Info("Start to rollback all webhook secrets") + err = refreshWebHookSecret(token) + if err != nil { + log.Errorf("Failed to refresh webhook secret, err: %s", err) + return err + } + + log.Info("Start to rollback codeHost type") + err = rollbackCodehostType() + if err != nil { + log.Errorf("Failed to rollback codehost type, err: %s", err) + return err + } + + return nil +} + +type hookRefresher interface { + RefreshWebHookSecret(secret, owner, repo, hookID string) error +} + +func refreshWebHookSecret(secret string) error { + hooks, err := mongodb.NewWebHookColl().List() + if err != nil { + log.Errorf("Failed to list webhooks, err: %s", err) + return err + } + + codeHosts, err := internalmongodb.NewCodehostColl().List() + if err != nil { + log.Errorf("Failed to list codehosts, err: %s", err) + return err + } + + for _, hook := range hooks { + var cl hookRefresher + ch := getCodeHostByAddressAndOwner(hook.Address, hook.Owner, codeHosts) + if ch == nil { + log.Warnf("Failed to get codehost for %s/%s", hook.Address, hook.Owner) + continue + } + + switch ch.Type { + case setting.SourceFromGithub: + cl = github.NewClient(ch.AccessToken, config.ProxyHTTPSAddr(), ch.EnableProxy) + case setting.SourceFromGitlab: + cl, err = gitlab.NewClient(ch.ID, ch.Address, ch.AccessToken, config.ProxyHTTPSAddr(), ch.EnableProxy) + if err != nil { + log.Warnf("Failed to create gitlab client, err: %s", err) + continue + } + default: + log.Warnf("Invalid type: %s", ch.Type) + continue + } + + if err = cl.RefreshWebHookSecret(secret, hook.Owner, hook.Repo, hook.HookID); err != nil { + log.Warnf("Failed to refresh webhook secret for hook %d in %s/%s, err: %s", hook.HookID, hook.Owner, hook.Repo, err) + continue + } + } + + return nil +} + +func getCodeHostByAddressAndOwner(address, owner string, all []*internalmodels.CodeHost) *internalmodels.CodeHost { + for _, one := range all { + if one.Address != address { + continue + } + // it is a limitation, we support only one gitlab account before + if one.Type == "gitlab" { + return one + } + if one.Namespace == owner { + return one + } + } + + return nil +} + +// change tls from string to bool type +func changeEmailHostType() error { + emailHosts, err := internalmongodb.NewEmailHostColl().List() + if err != nil { + log.Errorf("Failed to list emailhosts , err: %s", err) + return err + } + for _, v := range emailHosts { + if value, ok := v.IsTLS.(string); ok { + if err := internalmongodb.NewEmailHostColl().ChangeType(v.ID, value); err != nil { + log.Warnf("Failed to change id:%v istls:%s , err: %s", v.ID, value, err) + return err + } + } + } + return nil +} + +// rollback tls from bool to string type +func rollbackEmailhostType() error { + // get all codehosts + codeHosts, err := internalmongodb.NewEmailHostColl().List() + if err != nil { + log.Errorf("Failed to list codehosts, err: %s", err) + return err + } + for _, v := range codeHosts { + if value, ok := v.IsTLS.(bool); ok { + if err := internalmongodb.NewEmailHostColl().RollbackType(v.ID, value); err != nil { + log.Warnf("Failed to rollback id:%v istls:%v , err: %s", v.ID, value, err) + return err + } + } + } + return nil +} + +// change type "1,2,3,4" to "github,gitlab..." +func changeCodehostType() error { + // get all codehosts + codeHosts, err := internalmongodb.NewCodehostColl().List() + if err != nil { + log.Errorf("Failed to list codehosts, err: %s", err) + return err + } + var finalErr error + // change type to readable string + for _, v := range codeHosts { + if err := internalmongodb.NewCodehostColl().ChangeType(v.ID, v.Type); err != nil { + log.Warnf("Failed to change id:%d type:%s , err: %s", v.ID, v.Type, err) + finalErr = err + } + } + return finalErr +} + +// rollback type "github,gitlab..." to "1,2..." +func rollbackCodehostType() error { + // get all codehosts + codeHosts, err := internalmongodb.NewCodehostColl().List() + if err != nil { + log.Errorf("Failed to list codehosts, err: %s", err) + return err + } + var finalErr error + // rollback change type to readable string + for _, v := range codeHosts { + if err := internalmongodb.NewCodehostColl().RollbackType(v.ID, v.Type); err != nil { + log.Warnf("Failed to rollback id:%d type:%s , err: %s", v.ID, v.Type, err) + finalErr = err + continue + } + } + return finalErr +} + +func getWebHookTokenFromOrganization() string { + org, found, err := internalmongodb.NewOrganizationColl().Get(1) + if err != nil || !found { + log.Warnf("Failed to get organization, err: %s", err) + return "" + } + + return org.Token +} diff --git a/pkg/cli/upgradeassistant/cmd/migrate/180.go b/pkg/cli/upgradeassistant/cmd/migrate/180.go new file mode 100644 index 0000000000000000000000000000000000000000..7281b9312651a80e488f21c2038d0bcd0d857426 --- /dev/null +++ b/pkg/cli/upgradeassistant/cmd/migrate/180.go @@ -0,0 +1,194 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrate + +import ( + "context" + "fmt" + "time" + + "go.mongodb.org/mongo-driver/bson" + "k8s.io/apimachinery/pkg/util/sets" + + internalmodels "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/repository/models" + internalmongodb "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/repository/mongodb" + "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/upgradepath" + "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/policy/core/repository/models" + "github.com/koderover/zadig/pkg/microservice/policy/core/repository/mongodb" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/log" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +func init() { + upgradepath.RegisterHandler("1.7.1", "1.8.0", V171ToV180) + upgradepath.RegisterHandler("1.8.0", "1.7.1", V180ToV171) +} + +// V171ToV180 update all the roleBinding names in this format "{uid}-{roleName}-{roleNamespace}" +// Caution: this migration contains unrecoverable changes, please back up the database in advance +func V171ToV180() error { + log.Info("Migrating data from 1.7.1 to 1.8.0") + if err := updateAllRoleBindingNames(); err != nil { + log.Errorf("Failed to update roleBinding names, err: %s", err) + return err + } + + log.Info("Start to patchProductRegistryID") + if err := patchProductRegistryID(); err != nil { + log.Errorf("Failed to patchProductRegistryID, err: %s", err) + return err + } + + log.Info("Start to addProjectClusterRelation") + if err := initProjectClusterRelation(); err != nil { + log.Errorf("Failed to initProjectClusterRelation, err: %s", err) + return err + } + + return nil +} + +func V180ToV171() error { + log.Info("Rollback data from 1.8.0 to 1.7.1") + return nil +} + +func patchProductRegistryID() error { + // get all products + products, err := internalmongodb.NewProductColl().List(&internalmongodb.ProductListOptions{}) + if err != nil { + log.Errorf("Failed to list products, err: %s", err) + return err + } + if len(products) == 0 { + return nil + } + registry, err := internalmongodb.NewRegistryNamespaceColl().Find(&internalmongodb.FindRegOps{IsDefault: true}) + if err != nil { + log.Errorf("Failed to find default registry, err: %s", err) + return err + } + if registry == nil { + return nil + } + // change type to readable string + for _, v := range products { + if len(v.RegistryID) == 0 { + v.RegistryID = registry.ID.Hex() + } + } + err = internalmongodb.NewProductColl().UpdateAllRegistry(products) + if err != nil { + log.Errorf("Failed to update products, err: %s", err) + return err + } + return nil +} + +func updateAllRoleBindingNames() error { + coll := newRoleBindingColl() + rbs, err := coll.List() + if err != nil { + log.Errorf("Failed to list roleBindings, err: %s", err) + return err + } + + var lastErr error + for _, rb := range rbs { + newName := roleBindingName(rb) + if newName == "" || rb.Name == newName { + continue + } + + log.Infof("Update roleBinding name in namespace %s from %s to %s", rb.Namespace, rb.Name, newName) + if err = updateRoleBindingName(coll, rb, newName); err != nil { + log.Warnf("Failed to update roleBinding, err: %s", err) + lastErr = err + continue + } + } + + return lastErr +} + +func roleBindingName(rb *models.RoleBinding) string { + if len(rb.Subjects) != 1 || rb.Subjects[0].Kind != models.UserKind { + return "" + } + + return config.RoleBindingNameFromUIDAndRole(rb.Subjects[0].UID, setting.RoleType(rb.RoleRef.Name), rb.RoleRef.Namespace) +} + +func updateRoleBindingName(c *mongodb.RoleBindingColl, rb *models.RoleBinding, newName string) error { + query := bson.M{"name": rb.Name, "namespace": rb.Namespace} + + change := bson.M{"$set": bson.M{ + "name": newName, + }} + + _, err := c.UpdateOne(context.TODO(), query, change) + return err +} + +func newRoleBindingColl() *mongodb.RoleBindingColl { + name := models.RoleBinding{}.TableName() + return &mongodb.RoleBindingColl{ + Collection: mongotool.Database(fmt.Sprintf("%s_policy", config.MongoDatabase())).Collection(name), + } +} + +func initProjectClusterRelation() error { + projects, err := internalmongodb.NewProjectColl().List() + if err != nil { + log.Errorf("Failed to list projects, err: %s", err) + return err + } + clusters, err := internalmongodb.NewK8SClusterColl().List() + if err != nil { + log.Errorf("Failed to list clusters, err: %s", err) + return err + } + clusterIDs := sets.NewString() + for _, cluster := range clusters { + clusterIDs.Insert(cluster.ID.Hex()) + } + + // insert local cluster + if err = internalmongodb.NewK8SClusterColl().Create(&internalmodels.K8SCluster{ + Name: fmt.Sprintf("%s-%s", "local", time.Now().Format("20060102150405")), + Status: setting.Normal, + Local: true, + }, setting.LocalClusterID); err != nil { + log.Errorf("Failed to create local cluster, err: %s", err) + return err + } + clusterIDs.Insert(setting.LocalClusterID) + + for _, project := range projects { + for _, clusterID := range clusterIDs.List() { + if err := internalmongodb.NewProjectClusterRelationColl().Create(&internalmodels.ProjectClusterRelation{ + ProjectName: project.ProductName, + ClusterID: clusterID, + }); err != nil { + log.Warnf("Failed to create projectClusterRelation, err: %s", err) + } + } + } + return nil +} diff --git a/pkg/cli/upgradeassistant/cmd/migrate/migrate.go b/pkg/cli/upgradeassistant/cmd/migrate/migrate.go index cac8c4ab92bf94d1d928d29bd106990cab46b9bc..b07c59e1327e855d16cb63dd9be967c9d5cd0e68 100644 --- a/pkg/cli/upgradeassistant/cmd/migrate/migrate.go +++ b/pkg/cli/upgradeassistant/cmd/migrate/migrate.go @@ -31,7 +31,7 @@ import ( const oldServiceTemplateCounterName = "service:%s&type:%s" func init() { - upgradepath.AddHandler(upgradepath.V130, upgradepath.V131, V130ToV131) + upgradepath.RegisterHandler("1.3.0", "1.3.1", V130ToV131) // upgradepath.AddHandler(upgradepath.V131, upgradepath.V130, V131ToV130) } @@ -58,7 +58,7 @@ func V130ToV131() error { log.Errorf("Failed to get projects, err: %s", err) return err } - allEnvs, err := mongodb.NewProductColl().List(&mongodb.ProductListOptions{ExcludeStatus: setting.ProductStatusDeleting}) + allEnvs, err := mongodb.NewProductColl().List(&mongodb.ProductListOptions{ExcludeStatus: []string{setting.ProductStatusDeleting, setting.ProductStatusUnknown}}) if err != nil { log.Errorf("Failed to get envs, err: %s", err) return err @@ -157,7 +157,7 @@ func V131ToV130() error { log.Errorf("Failed to get projects, err: %s", err) return err } - allEnvs, err := mongodb.NewProductColl().List(&mongodb.ProductListOptions{ExcludeStatus: setting.ProductStatusDeleting}) + allEnvs, err := mongodb.NewProductColl().List(&mongodb.ProductListOptions{ExcludeStatus: []string{setting.ProductStatusDeleting, setting.ProductStatusUnknown}}) if err != nil { log.Errorf("Failed to get envs, err: %s", err) return err diff --git a/pkg/cli/upgradeassistant/internal/repository/models/basic_images.go b/pkg/cli/upgradeassistant/internal/repository/models/basic_images.go new file mode 100644 index 0000000000000000000000000000000000000000..0e624fa5adeaa63cf49d5c67ef1ec94ed5e86e0c --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/models/basic_images.go @@ -0,0 +1,44 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" +) + +const ( + ImageFromKoderover = "koderover" + ImageFromCustom = "custom" +) + +// BasicImage ... +type BasicImage struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + Value string `bson:"value" json:"value"` + Label string `bson:"label" json:"label"` + ImageFrom string `bson:"image_from" json:"image_from"` + CreateTime int64 `bson:"create_time" json:"create_time"` + UpdateTime int64 `bson:"update_time" json:"update_time"` + UpdateBy string `bson:"update_by" json:"update_by"` + // New field since 1.12 + // ImageType is the type of the image, currently only one type is supported called sonar + ImageType string `bson:"image_type" json:"image_type"` +} + +func (BasicImage) TableName() string { + return "basic_image" +} diff --git a/pkg/cli/upgradeassistant/internal/repository/models/build.go b/pkg/cli/upgradeassistant/internal/repository/models/build.go new file mode 100644 index 0000000000000000000000000000000000000000..780edeae773335295ae1cd1e11da0e572fb928f3 --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/models/build.go @@ -0,0 +1,60 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" + + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/types" +) + +type Build struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + Name string `bson:"name" json:"name"` + ProductName string `bson:"product_name" json:"product_name"` + PreBuild *PreBuild `bson:"pre_build" json:"pre_build"` + Timeout int `bson:"timeout" json:"timeout"` + // TODO: Deprecated. + Caches []string `bson:"caches" json:"caches"` + + // New since V1.10.0. + CacheEnable bool `bson:"cache_enable" json:"cache_enable"` + CacheDirType types.CacheDirType `bson:"cache_dir_type" json:"cache_dir_type"` + CacheUserDir string `bson:"cache_user_dir" json:"cache_user_dir"` + // New since V1.10.0. Only to tell the webpage should the advanced settings be displayed + AdvancedSettingsModified bool `bson:"advanced_setting_modified" json:"advanced_setting_modified"` + JenkinsBuild *JenkinsBuild `bson:"jenkins_build,omitempty" json:"jenkins_build,omitempty"` +} + +type JenkinsBuild struct { + JenkinsID string `bson:"jenkins_id" json:"jenkins_id"` + JobName string `bson:"job_name" json:"job_name"` + JenkinsBuildParam []*types.JenkinsBuildParam `bson:"jenkins_build_param" json:"jenkins_build_params"` +} + +// PreBuild prepares an environment for a job +type PreBuild struct { + ClusterID string `bson:"cluster_id" json:"cluster_id"` + ResReq setting.Request `bson:"res_req" json:"res_req"` + // TODO: Deprecated. + CleanWorkspace bool `bson:"clean_workspace" json:"clean_workspace"` +} + +func (Build) TableName() string { + return "module_build" +} diff --git a/pkg/cli/upgradeassistant/internal/repository/models/cluster.go b/pkg/cli/upgradeassistant/internal/repository/models/cluster.go new file mode 100644 index 0000000000000000000000000000000000000000..af5ac0f8cfda42c6393deb69e232dc3c6079e0dd --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/models/cluster.go @@ -0,0 +1,33 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/koderover/zadig/pkg/setting" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type K8SCluster struct { + ID primitive.ObjectID `json:"id,omitempty" bson:"_id,omitempty"` + Name string `json:"name" bson:"name"` + Status setting.K8SClusterStatus `json:"status" bson:"status"` + Local bool `json:"local" bson:"local"` +} + +func (K8SCluster) TableName() string { + return "k8s_cluster" +} diff --git a/pkg/cli/upgradeassistant/internal/repository/models/codehost.go b/pkg/cli/upgradeassistant/internal/repository/models/codehost.go new file mode 100644 index 0000000000000000000000000000000000000000..a712d145cd7951d6a14f4e63a26613e8e5a28164 --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/models/codehost.go @@ -0,0 +1,33 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +type CodeHost struct { + ID int `bson:"id" json:"id"` + Type string `bson:"type" json:"type"` + Address string `bson:"address" json:"address"` + Namespace string `bson:"namespace" json:"namespace"` + AccessToken string `bson:"access_token" json:"accessToken"` + EnableProxy bool `bson:"enable_proxy" json:"enable_proxy"` + Alias string `bson:"alias" json:"alias"` + Username string `bson:"username" json:"username"` + AccessKey string `bson:"application_id" json:"application_id"` +} + +func (CodeHost) TableName() string { + return "code_host" +} diff --git a/pkg/cli/upgradeassistant/internal/repository/models/emailhost.go b/pkg/cli/upgradeassistant/internal/repository/models/emailhost.go new file mode 100644 index 0000000000000000000000000000000000000000..dbf28fcbe01794706ac40849809cca1aeadbf611 --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/models/emailhost.go @@ -0,0 +1,37 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type EmailHost struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + Name string `bson:"name" json:"name"` + Port int `bson:"port" json:"port"` + Username string `bson:"username" json:"username"` + Password string `bson:"password" json:"password"` + IsTLS interface{} `bson:"is_tls" json:"isTLS"` + CreatedAt int64 `bson:"created_at" json:"created_at"` + DeletedAt int64 `bson:"deleted_at" json:"deleted_at"` + UpdatedAt int64 `bson:"updated_at" json:"updated_at"` +} + +func (EmailHost) TableName() string { + return "email_host" +} diff --git a/pkg/cli/upgradeassistant/internal/repository/models/env_configs.go b/pkg/cli/upgradeassistant/internal/repository/models/env_configs.go new file mode 100644 index 0000000000000000000000000000000000000000..f3ae9785c1585dd5c3f36197d0f4b7d7a80266c6 --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/models/env_configs.go @@ -0,0 +1,100 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type EnvResource struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + Type string `bson:"type" json:"type"` + ProductName string `bson:"product_name" json:"product_name"` + CreateTime int64 `bson:"create_time" json:"create_time"` + UpdateUserName string `bson:"update_user_name" json:"update_user_name"` + Namespace string `bson:"namespace,omitempty" json:"namespace,omitempty"` + Status string `bson:"status" json:"status"` + DeletedAt int64 `bson:"deleted_at" json:"deleted_at" ` + EnvName string `bson:"env_name" json:"env_name"` + Name string `bson:"name" json:"name"` + YamlData string `bson:"yaml_data" json:"yaml_data"` +} + +func (EnvResource) TableName() string { + return "env_resource" +} + +type EnvConfigMap struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + ProductName string `bson:"product_name" json:"product_name"` + CreateTime int64 `bson:"create_time" json:"create_time"` + UpdateUserName string `bson:"update_user_name" json:"update_user_name"` + Namespace string `bson:"namespace,omitempty" json:"namespace,omitempty"` + EnvName string `bson:"env_name" json:"env_name"` + Name string `bson:"name" json:"name"` + YamlData string `bson:"yaml_data" json:"yaml_data"` +} + +func (EnvConfigMap) TableName() string { + return "env_configmap" +} + +type EnvIngress struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + ProductName string `bson:"product_name" json:"product_name"` + CreateTime int64 `bson:"create_time" json:"create_time"` + UpdateUserName string `bson:"update_user_name" json:"update_user_name"` + Namespace string `bson:"namespace,omitempty" json:"namespace,omitempty"` + Status string `bson:"status" json:"status"` + EnvName string `bson:"env_name" json:"env_name"` + Name string `bson:"name" json:"name"` + YamlData string `bson:"yaml_data" json:"yaml_data"` +} + +func (EnvIngress) TableName() string { + return "env_ingress" +} + +type EnvPvc struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + ProductName string `bson:"product_name" json:"product_name"` + CreateTime int64 `bson:"create_time" json:"create_time"` + UpdateUserName string `bson:"update_user_name" json:"update_user_name"` + Namespace string `bson:"namespace,omitempty" json:"namespace,omitempty"` + Status string `bson:"status" json:"status"` + EnvName string `bson:"env_name" json:"env_name"` + Name string `bson:"name" json:"name"` + YamlData string `bson:"yaml_data" json:"yaml_data"` +} + +func (EnvPvc) TableName() string { + return "env_pvc" +} + +type EnvSecret struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + ProductName string `bson:"product_name" json:"product_name"` + CreateTime int64 `bson:"create_time" json:"create_time"` + UpdateUserName string `bson:"update_user_name" json:"update_user_name"` + Namespace string `bson:"namespace,omitempty" json:"namespace,omitempty"` + Status string `bson:"status" json:"status"` + EnvName string `bson:"env_name" json:"env_name"` + Name string `bson:"name" json:"name"` + YamlData string `bson:"yaml_data" json:"yaml_data"` +} + +func (EnvSecret) TableName() string { + return "env_secret" +} diff --git a/pkg/cli/upgradeassistant/internal/repository/models/jenkins.go b/pkg/cli/upgradeassistant/internal/repository/models/jenkins.go new file mode 100644 index 0000000000000000000000000000000000000000..96910307c511a551234da2750aa4bb5f3cfe3366 --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/models/jenkins.go @@ -0,0 +1,32 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type JenkinsIntegration struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + URL string `bson:"url" json:"url"` + Username string `bson:"username" json:"username"` + Password string `bson:"password" json:"password"` + UpdateBy string `bson:"update_by" json:"update_by"` + UpdatedAt int64 `bson:"updated_at" json:"updated_at"` +} + +func (j JenkinsIntegration) TableName() string { + return "jenkins_integration" +} diff --git a/pkg/cli/upgradeassistant/internal/repository/models/oldpolicy.go b/pkg/cli/upgradeassistant/internal/repository/models/oldpolicy.go new file mode 100644 index 0000000000000000000000000000000000000000..8ddeb087fea3a7146aa312fd4eff89ddd0f2fd76 --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/models/oldpolicy.go @@ -0,0 +1,84 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/koderover/zadig/pkg/setting" +) + +type PolicyMeta struct { + Resource string `bson:"resource" json:"resource"` + Alias string `bson:"alias" json:"alias"` + Description string `bson:"description" json:"description"` + Rules []*PolicyMetaRule `bson:"rules" json:"rules"` +} + +type Role struct { + Name string `bson:"name" json:"name"` + Namespace string `bson:"namespace" json:"namespace"` + Rules []*Rule `bson:"rules" json:"rules"` +} + +type Rule struct { + // Verbs is a list of http methods or resource actions that apply to ALL the Resources contained in this rule. '*' represents all methods. + Verbs []string `bson:"verbs" json:"verbs"` + Resources []string `bson:"resources" json:"resources"` + Kind string `bson:"kind" json:"kind"` + MatchAttributes []MatchAttribute `bson:"match_attributes" json:"match_attributes"` +} + +type MatchAttribute struct { + Key string `json:"key"` + Value string `json:"value"` +} + +type RoleBinding struct { + Name string `bson:"name" json:"name"` + Namespace string `bson:"namespace" json:"namespace"` + + // RoleRef can reference a namespaced or cluster scoped Role. + RoleRef *RoleRef `bson:"role_ref" json:"roleRef"` + Type setting.ResourceType `bson:"type" json:"type"` +} + +// RoleRef contains information that points to the role being used +type RoleRef struct { + Name string `bson:"name" json:"name"` + + // Namespace of the referenced object. if the object is cluster scoped, namespace is empty. + Namespace string `bson:"namespace" json:"namespace"` +} + +type PolicyMetaRule struct { + Action string `bson:"action" json:"action"` + Alias string `bson:"alias" json:"alias"` + Description string `bson:"description" json:"description"` + Rules []*ActionRule `bson:"rules" json:"rules"` +} + +type ActionRule struct { + Method string `bson:"method" json:"method"` + Endpoint string `bson:"endpoint" json:"endpoint"` + ResourceType string `bson:"resource_type,omitempty" json:"resourceType,omitempty"` + IDRegex string `bson:"id_regex,omitempty" json:"idRegex,omitempty"` + MatchAttributes []Attribute `bson:"match_attributes,omitempty" json:"matchAttributes,omitempty"` +} + +type Attribute struct { + Key string `bson:"key" json:"key"` + Value string `bson:"value" json:"value"` +} diff --git a/pkg/cli/upgradeassistant/internal/repository/models/organization.go b/pkg/cli/upgradeassistant/internal/repository/models/organization.go new file mode 100644 index 0000000000000000000000000000000000000000..fe5458327446cc8de93c761d44c960637681a4d5 --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/models/organization.go @@ -0,0 +1,26 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +type Organization struct { + ID int `bson:"id" json:"id"` + Token string `bson:"token,omitempty" json:"token,omitempty"` +} + +func (Organization) TableName() string { + return "organization" +} diff --git a/pkg/cli/upgradeassistant/internal/repository/models/product.go b/pkg/cli/upgradeassistant/internal/repository/models/product.go new file mode 100644 index 0000000000000000000000000000000000000000..dac991e64f39c46de85af0a626b309aac77e1e15 --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/models/product.go @@ -0,0 +1,133 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" + + templatemodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" +) + +type ProductAuthType string +type ProductPermission string + +// Vars do not save, only input parameters +type Product struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + ProductName string `bson:"product_name" json:"product_name"` + CreateTime int64 `bson:"create_time" json:"create_time"` + UpdateTime int64 `bson:"update_time" json:"update_time"` + Namespace string `bson:"namespace,omitempty" json:"namespace,omitempty"` + Status string `bson:"status" json:"status"` + Revision int64 `bson:"revision" json:"revision"` + Enabled bool `bson:"enabled" json:"enabled"` + EnvName string `bson:"env_name" json:"env_name"` + UpdateBy string `bson:"update_by" json:"update_by"` + Auth []*ProductAuth `bson:"auth" json:"auth"` + Visibility string `bson:"-" json:"visibility"` + Services [][]*ProductService `bson:"services" json:"services"` + Render *RenderInfo `bson:"render" json:"render"` + Error string `bson:"error" json:"error"` + Vars []*templatemodels.RenderKV `bson:"vars,omitempty" json:"vars,omitempty"` + ChartInfos []*templatemodels.RenderChart `bson:"-" json:"chart_infos,omitempty"` + IsPublic bool `bson:"is_public" json:"isPublic"` + RoleIDs []int64 `bson:"role_ids" json:"roleIds"` + ClusterID string `bson:"cluster_id,omitempty" json:"cluster_id,omitempty"` + RecycleDay int `bson:"recycle_day" json:"recycle_day"` + Source string `bson:"source" json:"source"` + IsOpenSource bool `bson:"is_opensource" json:"is_opensource"` + RegistryID string `bson:"registry_id" json:"registry_id"` + IsForkedProduct bool `bson:"-" json:"-"` +} + +type RenderInfo struct { + Name string `bson:"name" json:"name"` + Revision int64 `bson:"revision" json:"revision"` + ProductTmpl string `bson:"product_tmpl" json:"product_tmpl"` + Description string `bson:"description" json:"description"` +} + +type ProductAuth struct { + Type ProductAuthType `bson:"type" json:"type"` + Name string `bson:"name" json:"name"` + Permissions []ProductPermission `bson:"permissions" json:"permissions"` +} + +// ImagePathSpec paths in yaml used to parse image +type ImagePathSpec struct { + Repo string `bson:"repo,omitempty" json:"repo,omitempty"` + Image string `bson:"image,omitempty" json:"image,omitempty"` + Tag string `bson:"tag,omitempty" json:"tag,omitempty"` +} + +type Container struct { + Name string `bson:"name" json:"name"` + Image string `bson:"image" json:"image"` + ImagePath *ImagePathSpec `bson:"image_path,omitempty" json:"imagePath,omitempty"` +} + +type EnvConfig struct { + EnvName string `bson:"env_name,omitempty" json:"env_name"` + HostIDs []string `bson:"host_ids,omitempty" json:"host_ids"` + Labels []string `bson:"labels,omitempty" json:"labels"` +} + +type ProductService struct { + ServiceName string `bson:"service_name" json:"service_name"` + ProductName string `bson:"product_name" json:"product_name"` + Type string `bson:"type" json:"type"` + Revision int64 `bson:"revision" json:"revision"` + Containers []*Container `bson:"containers" json:"containers,omitempty"` + Render *RenderInfo `bson:"render,omitempty" json:"render,omitempty"` // Record the render information of each service to facilitate the update of a single service + EnvConfigs []*EnvConfig `bson:"-" json:"env_configs,omitempty"` +} + +type ServiceConfig struct { + ConfigName string `bson:"config_name" json:"config_name"` + Revision int64 `bson:"revision" json:"revision"` +} + +func (Product) TableName() string { + return "product" +} + +func (p *Product) GetNamespace() string { + return p.ProductName + "-env-" + p.EnvName +} + +func (p *Product) GetGroupServiceNames() [][]string { + var resp [][]string + for _, group := range p.Services { + services := make([]string, 0, len(group)) + for _, service := range group { + services = append(services, service.ServiceName) + } + resp = append(resp, services) + } + return resp +} + +func (p *Product) GetServiceMap() map[string]*ProductService { + ret := make(map[string]*ProductService) + for _, group := range p.Services { + for _, svc := range group { + ret[svc.ServiceName] = svc + } + } + + return ret +} diff --git a/pkg/cli/upgradeassistant/internal/repository/models/project.go b/pkg/cli/upgradeassistant/internal/repository/models/project.go new file mode 100644 index 0000000000000000000000000000000000000000..33108cf38e7ba926672d235397a9542c978e02dc --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/models/project.go @@ -0,0 +1,25 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +type Project struct { + ProductName string `bson:"product_name" json:"product_name"` +} + +func (Project) TableName() string { + return "template_product" +} diff --git a/pkg/cli/upgradeassistant/internal/repository/models/project_cluster_relation.go b/pkg/cli/upgradeassistant/internal/repository/models/project_cluster_relation.go new file mode 100644 index 0000000000000000000000000000000000000000..57d8fcd4d96350de5af89e860cdab43c24f89889 --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/models/project_cluster_relation.go @@ -0,0 +1,33 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type ProjectClusterRelation struct { + ID primitive.ObjectID `json:"id,omitempty" bson:"_id,omitempty"` + ProjectName string `json:"project_name" bson:"project_name"` + ClusterID string `json:"cluster_id" bson:"cluster_id"` + CreatedAt int64 `json:"createdAt" bson:"createdAt"` + CreatedBy string `json:"createdBy" bson:"createdBy"` +} + +func (ProjectClusterRelation) TableName() string { + return "project_cluster_relation" +} diff --git a/pkg/cli/upgradeassistant/internal/repository/models/registry_namespace.go b/pkg/cli/upgradeassistant/internal/repository/models/registry_namespace.go new file mode 100644 index 0000000000000000000000000000000000000000..6a159859ef4025d305b5c32391f6516b23533292 --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/models/registry_namespace.go @@ -0,0 +1,57 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "errors" + + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type RegistryNamespace struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + RegAddr string `bson:"reg_addr" json:"reg_addr"` + RegType string `bson:"reg_type" json:"reg_type"` + RegProvider string `bson:"reg_provider" json:"reg_provider"` + IsDefault bool `bson:"is_default" json:"is_default"` + Namespace string `bson:"namespace" json:"namespace"` + AccessKey string `bson:"access_key" json:"access_key"` + SecretKey string `bson:"secret_key" json:"secret_key"` + Region string `bson:"region,omitempty" json:"region,omitempty"` + UpdateTime int64 `bson:"update_time" json:"update_time"` + UpdateBy string `bson:"update_by" json:"update_by"` +} + +func (ns *RegistryNamespace) Validate() error { + if ns.RegAddr == "" { + return errors.New("empty reg_addr") + } + + if ns.RegProvider == "" { + return errors.New("empty reg_provider") + } + + if ns.Namespace == "" { + return errors.New("empty namespace") + } + + return nil +} + +func (RegistryNamespace) TableName() string { + return "registry_namespace" +} diff --git a/pkg/cli/upgradeassistant/internal/repository/models/testing.go b/pkg/cli/upgradeassistant/internal/repository/models/testing.go new file mode 100644 index 0000000000000000000000000000000000000000..204f8f7079761d2dee48a9d076e8748323a45cee --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/models/testing.go @@ -0,0 +1,119 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/types" +) + +type Testing struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + PreTest *PreTest `bson:"pre_test" json:"pre_test"` + Timeout int `bson:"timeout" json:"timeout"` + HookCtl *TestingHookCtrl `bson:"hook_ctl" json:"hook_ctl"` + NotifyCtl *NotifyCtl `bson:"notify_ctl,omitempty" json:"notify_ctl,omitempty"` + NotifyCtls []*NotifyCtl `bson:"notify_ctls" json:"notify_ctls"` + + Schedules *ScheduleCtrl `bson:"schedules,omitempty" json:"schedules,omitempty"` + + ArtifactPaths []string `bson:"artifact_paths,omitempty" json:"artifact_paths,omitempty"` + + // TODO: Deprecated. + Caches []string `bson:"caches" json:"caches"` + + // New since V1.10.0. + CacheEnable bool `bson:"cache_enable" json:"cache_enable"` + CacheDirType types.CacheDirType `bson:"cache_dir_type" json:"cache_dir_type"` + CacheUserDir string `bson:"cache_user_dir" json:"cache_user_dir"` + // New since V1.10.0. Only to tell the webpage should the advanced settings be displayed + AdvancedSettingsModified bool `bson:"advanced_setting_modified" json:"advanced_setting_modified"` +} + +// PreTest prepares an environment for a job +type PreTest struct { + ClusterID string `bson:"cluster_id" json:"cluster_id"` + ResReq setting.Request `bson:"res_req" json:"res_req"` + // TODO: Deprecated. + CleanWorkspace bool `bson:"clean_workspace" json:"clean_workspace"` +} + +type TestingHookCtrl struct { + Enabled bool `bson:"enabled" json:"enabled"` + Items []*TestingHook `bson:"items" json:"items"` +} + +type ScheduleCtrl struct { + Enabled bool `bson:"enabled" json:"enabled"` + //Items []*Schedule `bson:"items" json:"items"` +} + +type NotifyCtl struct { + Enabled bool `bson:"enabled" json:"enabled"` + WebHookType string `bson:"webhook_type" json:"webhook_type"` + WeChatWebHook string `bson:"weChat_webHook,omitempty" json:"weChat_webHook,omitempty"` + DingDingWebHook string `bson:"dingding_webhook,omitempty" json:"dingding_webhook,omitempty"` + FeiShuWebHook string `bson:"feishu_webhook,omitempty" json:"feishu_webhook,omitempty"` + AtMobiles []string `bson:"at_mobiles,omitempty" json:"at_mobiles,omitempty"` + IsAtAll bool `bson:"is_at_all,omitempty" json:"is_at_all,omitempty"` + NotifyTypes []string `bson:"notify_type" json:"notify_type"` +} + +type TestingHook struct { + AutoCancel bool `bson:"auto_cancel" json:"auto_cancel"` + MainRepo *MainHookRepo `bson:"main_repo" json:"main_repo"` + TestArgs *TestTaskArgs `bson:"test_args" json:"test_args"` +} + +type MainHookRepo struct { + Name string `bson:"name,omitempty" json:"name,omitempty"` + Description string `bson:"description,omitempty" json:"description,omitempty"` + Source string `bson:"source,omitempty" json:"source,omitempty"` + RepoOwner string `bson:"repo_owner" json:"repo_owner"` + RepoName string `bson:"repo_name" json:"repo_name"` + Branch string `bson:"branch" json:"branch"` + Tag string `bson:"tag" json:"tag"` + Committer string `bson:"committer" json:"committer"` + MatchFolders []string `bson:"match_folders" json:"match_folders,omitempty"` + CodehostID int `bson:"codehost_id" json:"codehost_id"` + Events []config.HookEventType `bson:"events" json:"events"` + Label string `bson:"label" json:"label"` + Revision string `bson:"revision" json:"revision"` + IsRegular bool `bson:"is_regular" json:"is_regular"` +} + +type TestTaskArgs struct { + ProductName string `bson:"product_name" json:"product_name"` + TestName string `bson:"test_name" json:"test_name"` + TestTaskCreator string `bson:"test_task_creator" json:"test_task_creator"` + NotificationID string `bson:"notification_id" json:"notification_id"` + ReqID string `bson:"req_id" json:"req_id"` + // webhook触发测试任务时，触发任务的repo、prID和commitID + MergeRequestID string `bson:"merge_request_id" json:"merge_request_id"` + CommitID string `bson:"commit_id" json:"commit_id"` + Source string `bson:"source" json:"source"` + CodehostID int `bson:"codehost_id" json:"codehost_id"` + RepoOwner string `bson:"repo_owner" json:"repo_owner"` + RepoName string `bson:"repo_name" json:"repo_name"` +} + +func (Testing) TableName() string { + return "module_testing" +} diff --git a/pkg/cli/upgradeassistant/internal/repository/models/workflow.go b/pkg/cli/upgradeassistant/internal/repository/models/workflow.go new file mode 100644 index 0000000000000000000000000000000000000000..c543538f3c20859f5129ad9d2fb0faca059d6f1e --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/models/workflow.go @@ -0,0 +1,51 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type Workflow struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + Name string `bson:"name" json:"name"` + Enabled bool `bson:"enabled" json:"enabled"` + ProductTmplName string `bson:"product_tmpl_name" json:"product_tmpl_name"` + Team string `bson:"team,omitempty" json:"team,omitempty"` + EnvName string `bson:"env_name" json:"env_name,omitempty"` + Description string `bson:"description,omitempty" json:"description,omitempty"` + UpdateBy string `bson:"update_by" json:"update_by,omitempty"` + CreateBy string `bson:"create_by" json:"create_by,omitempty"` + UpdateTime int64 `bson:"update_time" json:"update_time"` + CreateTime int64 `bson:"create_time" json:"create_time"` + Schedules *ScheduleCtrl `bson:"schedules,omitempty" json:"schedules,omitempty"` + ScheduleEnabled bool `bson:"schedule_enabled" json:"schedule_enabled"` + // TODO: Deprecated. + NotifyCtl *NotifyCtl `bson:"notify_ctl,omitempty" json:"notify_ctl,omitempty"` + // New since V1.12.0. + NotifyCtls []*NotifyCtl `bson:"notify_ctls" json:"notify_ctls"` + BaseName string `bson:"base_name" json:"base_name"` + + // ResetImage indicate whether reset image to original version after completion + ResetImage bool `bson:"reset_image" json:"reset_image"` + // IsParallel 控制单一工作流的任务是否支持并行处理 + IsParallel bool `json:"is_parallel" bson:"is_parallel"` +} + +func (Workflow) TableName() string { + return "workflow" +} diff --git a/pkg/cli/upgradeassistant/internal/repository/mongodb/basic_imnages.go b/pkg/cli/upgradeassistant/internal/repository/mongodb/basic_imnages.go new file mode 100644 index 0000000000000000000000000000000000000000..2af43534b878c31ba1ad4b11b75bbf1b36acd632 --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/mongodb/basic_imnages.go @@ -0,0 +1,81 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "time" +) + +type BasicImageOpt struct { + Value string `bson:"value"` + Label string `bson:"label"` + ImageFrom string `bson:"image_from"` + ImageType string `bson:"image_type"` +} + +type BasicImageColl struct { + *mongo.Collection + + coll string +} + +func NewBasicImageColl() *BasicImageColl { + name := models.BasicImage{}.TableName() + coll := &BasicImageColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} + + return coll +} + +func (c *BasicImageColl) GetSonarTypeImage() ([]*models.BasicImage, error) { + query := bson.M{} + query["image_type"] = "sonar" + + ctx := context.Background() + resp := make([]*models.BasicImage, 0) + + cursor, err := c.Collection.Find(ctx, query) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &resp) + if err != nil { + return nil, err + } + + return resp, err +} + +func (c *BasicImageColl) CreateZadigSonarImage() error { + args := &models.BasicImage{ + Value: "sonarsource/sonar-scanner-cli", + Label: "sonar:latest", + CreateTime: time.Now().Unix(), + UpdateTime: time.Now().Unix(), + UpdateBy: "system", + ImageType: "sonar", + } + + _, err := c.InsertOne(context.TODO(), args) + return err +} diff --git a/pkg/cli/upgradeassistant/internal/repository/mongodb/build.go b/pkg/cli/upgradeassistant/internal/repository/mongodb/build.go new file mode 100644 index 0000000000000000000000000000000000000000..89acb248e177698c2964855328ac8510bc6fec1a --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/mongodb/build.go @@ -0,0 +1,83 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "errors" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/config" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type BuildListOption struct { +} + +type BuildColl struct { + *mongo.Collection + + coll string +} + +func NewBuildColl() *BuildColl { + name := models.Build{}.TableName() + return &BuildColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} +} + +func (c *BuildColl) List(opt *BuildListOption) ([]*models.Build, error) { + if opt == nil { + return nil, errors.New("nil ListOption") + } + + query := bson.M{} + + var resp []*models.Build + ctx := context.Background() + opts := options.Find() + cursor, err := c.Collection.Find(ctx, query, opts) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &resp) + if err != nil { + return nil, err + } + + return resp, nil +} + +func (c *BuildColl) Update(build *models.Build) error { + if build == nil { + return errors.New("nil Module args") + } + + query := bson.M{"name": build.Name} + if build.ProductName != "" { + query["product_name"] = build.ProductName + } + + updateBuild := bson.M{"$set": build} + + _, err := c.Collection.UpdateOne(context.TODO(), query, updateBuild) + return err +} diff --git a/pkg/cli/upgradeassistant/internal/repository/mongodb/cluster.go b/pkg/cli/upgradeassistant/internal/repository/mongodb/cluster.go new file mode 100644 index 0000000000000000000000000000000000000000..444b57bdcf884695373d7055987a4663b7ff6834 --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/mongodb/cluster.go @@ -0,0 +1,91 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + + models "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/config" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type K8SClusterColl struct { + *mongo.Collection + + coll string +} + +func NewK8SClusterColl() *K8SClusterColl { + name := models.K8SCluster{}.TableName() + return &K8SClusterColl{ + Collection: mongotool.Database(config.MongoDatabase()).Collection(name), + coll: name, + } +} + +func (c *K8SClusterColl) GetCollectionName() string { + return c.coll +} + +func (c *K8SClusterColl) List() ([]*models.K8SCluster, error) { + var clusters []*models.K8SCluster + + query := bson.M{} + cursor, err := c.Collection.Find(context.TODO(), query) + if err != nil { + if err == mongo.ErrNoDocuments { + return nil, nil + } + return nil, err + } + err = cursor.All(context.TODO(), &clusters) + if err != nil { + return nil, err + } + + return clusters, nil +} + +func (c *K8SClusterColl) Create(cluster *models.K8SCluster, id string) error { + if id != "" { + cluster.ID, _ = primitive.ObjectIDFromHex(id) + // If the local cluster already exists, do not insert，and return nil + if _, err := c.Get(id); err == nil { + return nil + } + } + _, err := c.InsertOne(context.TODO(), cluster) + return err +} + +func (c *K8SClusterColl) Get(id string) (*models.K8SCluster, error) { + oid, err := primitive.ObjectIDFromHex(id) + if err != nil { + return nil, err + } + + query := bson.M{"_id": oid} + res := &models.K8SCluster{} + err = c.FindOne(context.TODO(), query).Decode(res) + + return res, err +} diff --git a/pkg/cli/upgradeassistant/internal/repository/mongodb/codehost.go b/pkg/cli/upgradeassistant/internal/repository/mongodb/codehost.go new file mode 100644 index 0000000000000000000000000000000000000000..2bf1fa578a3167b2c7fd9370913e369d37f66340 --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/mongodb/codehost.go @@ -0,0 +1,118 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + + "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/repository/models" + "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/tool/log" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +func (c *CodehostColl) ChangeType(ID int, sourceType string) error { + query := bson.M{"id": ID} + + preType := sourceType + if sourceType == "1" { + sourceType = "gitlab" + } else if sourceType == "2" { + sourceType = "github" + } else if sourceType == "3" { + sourceType = "gerrit" + } else if sourceType == "4" { + sourceType = "codehub" + } else { + return nil + } + + change := bson.M{"$set": bson.M{ + "type": sourceType, + }} + _, err := c.Collection.UpdateOne(context.TODO(), query, change) + if err != nil { + log.Errorf("repository update fail,err:%s", err) + return err + } + log.Infof("success to change id:%d type:%s to type:%s", ID, preType, sourceType) + return nil +} + +func (c *CodehostColl) RollbackType(ID int, sourceType string) error { + query := bson.M{"id": ID} + + if sourceType == "gitlab" { + sourceType = "1" + } else if sourceType == "github" { + sourceType = "2" + } else if sourceType == "gerrit" { + sourceType = "3" + } else if sourceType == "codehub" { + sourceType = "4" + } else { + return nil + } + + change := bson.M{"$set": bson.M{ + "type": sourceType, + }} + _, err := c.Collection.UpdateOne(context.TODO(), query, change) + if err != nil { + log.Errorf("repository update fail,err:%s", err) + return err + } + log.Infof("success to rollback id:%d to type:%s", ID, sourceType) + return nil +} + +func (c *CodehostColl) List() ([]*models.CodeHost, error) { + codeHosts := make([]*models.CodeHost, 0) + query := bson.M{"deleted_at": 0} + cursor, err := c.Collection.Find(context.TODO(), query) + if err != nil { + return nil, err + } + err = cursor.All(context.TODO(), &codeHosts) + if err != nil { + return nil, err + } + + return codeHosts, nil +} + +type CodehostColl struct { + *mongo.Collection + + coll string +} + +type ListArgs struct { + Owner string + Address string + Source string +} + +func NewCodehostColl() *CodehostColl { + name := models.CodeHost{}.TableName() + coll := &CodehostColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} + + return coll +} diff --git a/pkg/cli/upgradeassistant/internal/repository/mongodb/emailhost.go b/pkg/cli/upgradeassistant/internal/repository/mongodb/emailhost.go new file mode 100644 index 0000000000000000000000000000000000000000..6c68464ebc8c78af6d6981bbabb505cf3b8192d7 --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/mongodb/emailhost.go @@ -0,0 +1,94 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + + "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/repository/models" + "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/tool/log" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +func NewEmailHostColl() *EmailHostColl { + name := models.EmailHost{}.TableName() + coll := &EmailHostColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} + + return coll +} + +type EmailHostColl struct { + *mongo.Collection + + coll string +} + +func (c *EmailHostColl) List() ([]*models.EmailHost, error) { + emailhosts := make([]*models.EmailHost, 0) + query := bson.M{"deleted_at": 0} + cursor, err := c.Collection.Find(context.TODO(), query) + if err != nil { + return nil, err + } + err = cursor.All(context.TODO(), &emailhosts) + if err != nil { + return nil, err + } + + return emailhosts, nil +} + +func (c *EmailHostColl) ChangeType(ID primitive.ObjectID, isTLS string) error { + query := bson.M{"_id": ID} + changedTLS := false + if isTLS == "2" { + changedTLS = true + } + change := bson.M{"$set": bson.M{ + "is_tls": changedTLS, + }} + _, err := c.Collection.UpdateOne(context.TODO(), query, change) + if err != nil { + log.Errorf("emailhost update fail,err:%s", err) + return err + } + log.Infof("success to change id:%v isTLS:%s to %v", ID, isTLS, changedTLS) + return nil +} + +func (c *EmailHostColl) RollbackType(ID primitive.ObjectID, isTLS bool) error { + query := bson.M{"_id": ID} + changedTLS := "1" + if isTLS == true { + changedTLS = "2" + } + change := bson.M{"$set": bson.M{ + "is_tls": changedTLS, + }} + _, err := c.Collection.UpdateOne(context.TODO(), query, change) + if err != nil { + log.Errorf("emailhost update fail,err:%s", err) + return err + } + log.Infof("success to change id:%v isTLS:%v to %s", ID, isTLS, changedTLS) + return nil +} diff --git a/pkg/cli/upgradeassistant/internal/repository/mongodb/env_resources.go b/pkg/cli/upgradeassistant/internal/repository/mongodb/env_resources.go new file mode 100644 index 0000000000000000000000000000000000000000..7454f869725a85efaf0200995dabb96622926321 --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/mongodb/env_resources.go @@ -0,0 +1,171 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/config" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type EnvResourceColl struct { + *mongo.Collection + coll string +} + +func NewEnvResourceColl() *EnvResourceColl { + name := models.EnvResource{}.TableName() + return &EnvResourceColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} +} + +func (c *EnvResourceColl) BatchInsert(resources []*models.EnvResource) error { + if len(resources) == 0 { + return nil + } + resList := make([]interface{}, 0, len(resources)) + for _, res := range resources { + resList = append(resList, res) + } + _, err := c.InsertMany(context.TODO(), resList) + return err +} + +type ConfigMapColl struct { + *mongo.Collection + coll string +} + +func NewConfigMapColl() *ConfigMapColl { + name := models.EnvConfigMap{}.TableName() + return &ConfigMapColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} +} + +func (c *ConfigMapColl) GetCollectionName() string { + return c.coll +} + +func (c *ConfigMapColl) List() ([]*models.EnvConfigMap, error) { + query := bson.M{} + + var resp []*models.EnvConfigMap + ctx := context.Background() + opts := options.Find() + + cursor, err := c.Collection.Find(ctx, query, opts) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &resp) + return resp, err +} + +type IngressColl struct { + *mongo.Collection + + coll string +} + +func NewIngressColl() *IngressColl { + name := models.EnvIngress{}.TableName() + return &IngressColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} +} + +func (c *IngressColl) GetCollectionName() string { + return c.coll +} + +func (c *IngressColl) List() ([]*models.EnvIngress, error) { + query := bson.M{} + + var resp []*models.EnvIngress + ctx := context.Background() + opts := options.Find() + cursor, err := c.Collection.Find(ctx, query, opts) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &resp) + return resp, err +} + +type PvcColl struct { + *mongo.Collection + + coll string +} + +func NewPvcColl() *PvcColl { + name := models.EnvPvc{}.TableName() + return &PvcColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} +} + +func (c *PvcColl) GetCollectionName() string { + return c.coll +} + +func (c *PvcColl) List() ([]*models.EnvPvc, error) { + query := bson.M{} + + var resp []*models.EnvPvc + ctx := context.Background() + opts := options.Find() + cursor, err := c.Collection.Find(ctx, query, opts) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &resp) + return resp, err +} + +type SecretColl struct { + *mongo.Collection + + coll string +} + +func NewSecretColl() *SecretColl { + name := models.EnvSecret{}.TableName() + return &SecretColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} +} + +func (c *SecretColl) GetCollectionName() string { + return c.coll +} + +func (c *SecretColl) List() ([]*models.EnvSecret, error) { + query := bson.M{} + + var resp []*models.EnvSecret + ctx := context.Background() + opts := options.Find() + cursor, err := c.Collection.Find(ctx, query, opts) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &resp) + return resp, err +} diff --git a/pkg/cli/upgradeassistant/internal/repository/mongodb/jenkins.go b/pkg/cli/upgradeassistant/internal/repository/mongodb/jenkins.go new file mode 100644 index 0000000000000000000000000000000000000000..3932f3145ad4787046a43e4d28cd2d3ebd33f474 --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/mongodb/jenkins.go @@ -0,0 +1,62 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/config" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type JenkinsIntegrationColl struct { + *mongo.Collection + + coll string +} + +func NewJenkinsIntegrationColl() *JenkinsIntegrationColl { + name := models.JenkinsIntegration{}.TableName() + coll := &JenkinsIntegrationColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} + + return coll +} + +func (c *JenkinsIntegrationColl) List() ([]*models.JenkinsIntegration, error) { + resp := make([]*models.JenkinsIntegration, 0) + query := bson.M{} + + ctx := context.Background() + opts := options.Find().SetSort(bson.D{{"updated_at", -1}}) + + cursor, err := c.Collection.Find(ctx, query, opts) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &resp) + if err != nil { + return nil, err + } + + return resp, nil +} diff --git a/pkg/cli/upgradeassistant/internal/repository/mongodb/organization.go b/pkg/cli/upgradeassistant/internal/repository/mongodb/organization.go new file mode 100644 index 0000000000000000000000000000000000000000..56a80822966dffa43f892ce315d780daace8c1f0 --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/mongodb/organization.go @@ -0,0 +1,56 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + + "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/repository/models" + "github.com/koderover/zadig/pkg/config" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type OrganizationColl struct { + *mongo.Collection + + coll string +} + +func NewOrganizationColl() *OrganizationColl { + name := models.Organization{}.TableName() + return &OrganizationColl{ + Collection: mongotool.Database(config.MongoDatabase()).Collection(name), + coll: name, + } +} + +func (c *OrganizationColl) Get(id int) (*models.Organization, bool, error) { + org := &models.Organization{} + query := bson.M{"id": id} + + err := c.FindOne(context.TODO(), query).Decode(org) + if err != nil { + if err == mongo.ErrNoDocuments { + return nil, false, nil + } + return nil, false, err + } + return org, true, nil +} diff --git a/pkg/cli/upgradeassistant/internal/repository/mongodb/product.go b/pkg/cli/upgradeassistant/internal/repository/mongodb/product.go new file mode 100644 index 0000000000000000000000000000000000000000..d88cb2a959b72f0dbb1524524ea1a511b8bb1d9a --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/mongodb/product.go @@ -0,0 +1,212 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type ProductFindOptions struct { + Name string + EnvName string + Namespace string +} + +// ClusterId is a primitive.ObjectID{}.Hex() +type ProductListOptions struct { + EnvName string + Name string + IsPublic bool + ClusterID string + IsSortByUpdateTime bool + IsSortByProductName bool + ExcludeStatus string + ExcludeSource string + Source string + InProjects []string + InEnvs []string +} + +type projectEnvs struct { + ID projectID `bson:"_id"` + ProjectName string `bson:"project_name"` + Envs []string `bson:"envs"` +} + +type projectID struct { + ProductName string `bson:"product_name"` +} + +type ProductColl struct { + *mongo.Collection + + coll string +} + +func NewProductColl() *ProductColl { + name := models.Product{}.TableName() + return &ProductColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} +} + +func (c *ProductColl) GetCollectionName() string { + return c.coll +} + +func (c *ProductColl) EnsureIndex(ctx context.Context) error { + mod := []mongo.IndexModel{ + { + Keys: bson.D{ + bson.E{Key: "env_name", Value: 1}, + bson.E{Key: "product_name", Value: 1}, + }, + Options: options.Index().SetUnique(true), + }, + { + Keys: bson.D{ + bson.E{Key: "env_name", Value: 1}, + bson.E{Key: "product_name", Value: 1}, + bson.E{Key: "update_time", Value: 1}, + }, + Options: options.Index().SetUnique(false), + }, + } + + _, err := c.Indexes().CreateMany(ctx, mod) + + return err +} + +type ProductEnvFindOptions struct { + Name string + Namespace string +} + +func (c *ProductColl) FindEnv(opt *ProductEnvFindOptions) (*models.Product, error) { + query := bson.M{} + if opt.Name != "" { + query["product_name"] = opt.Name + } + + if opt.Namespace != "" { + query["namespace"] = opt.Namespace + } + + ret := new(models.Product) + err := c.FindOne(context.TODO(), query).Decode(ret) + return ret, err +} + +func (c *ProductColl) Find(opt *ProductFindOptions) (*models.Product, error) { + res := &models.Product{} + query := bson.M{} + if opt.Name != "" { + query["product_name"] = opt.Name + } + if opt.EnvName != "" { + query["env_name"] = opt.EnvName + } + if opt.Namespace != "" { + query["namespace"] = opt.Namespace + } + + err := c.FindOne(context.TODO(), query).Decode(res) + return res, err +} + +func (c *ProductColl) List(opt *ProductListOptions) ([]*models.Product, error) { + var ret []*models.Product + query := bson.M{} + + if opt == nil { + opt = &ProductListOptions{} + } + if opt.EnvName != "" { + query["env_name"] = opt.EnvName + } else if len(opt.InEnvs) > 0 { + query["env_name"] = bson.M{"$in": opt.InEnvs} + } + if opt.Name != "" { + query["product_name"] = opt.Name + } + if opt.IsPublic { + query["is_public"] = opt.IsPublic + } + if opt.ClusterID != "" { + query["cluster_id"] = opt.ClusterID + } + if opt.Source != "" { + query["source"] = opt.Source + } + if opt.ExcludeSource != "" { + query["source"] = bson.M{"$ne": opt.ExcludeSource} + } + if opt.ExcludeStatus != "" { + query["status"] = bson.M{"$ne": opt.ExcludeStatus} + } + if len(opt.InProjects) > 0 { + query["product_name"] = bson.M{"$in": opt.InProjects} + } + + ctx := context.Background() + opts := options.Find() + if opt.IsSortByUpdateTime { + opts.SetSort(bson.D{{"update_time", -1}}) + } + if opt.IsSortByProductName { + opts.SetSort(bson.D{{"product_name", 1}}) + } + cursor, err := c.Collection.Find(ctx, query, opts) + if err != nil { + if err == mongo.ErrNoDocuments { + return nil, nil + } + return nil, err + } + + err = cursor.All(ctx, &ret) + if err != nil { + return nil, err + } + + return ret, nil +} + +func (c *ProductColl) UpdateAllRegistry(envs []*models.Product) error { + if len(envs) == 0 { + return nil + } + + var ms []mongo.WriteModel + for _, env := range envs { + ms = append(ms, + mongo.NewUpdateOneModel(). + SetFilter(bson.D{{"_id", env.ID}}). + SetUpdate(bson.D{{"$set", bson.D{{"registry_id", env.RegistryID}}}}), + ) + } + _, err := c.BulkWrite(context.TODO(), ms) + + return err +} diff --git a/pkg/cli/upgradeassistant/internal/repository/mongodb/project.go b/pkg/cli/upgradeassistant/internal/repository/mongodb/project.go new file mode 100644 index 0000000000000000000000000000000000000000..b596036d471afdfc918992b5ff3924b5c4de3842 --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/mongodb/project.go @@ -0,0 +1,61 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + + models "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/config" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type ProjectColl struct { + *mongo.Collection + + coll string +} + +func NewProjectColl() *ProjectColl { + name := models.Project{}.TableName() + return &ProjectColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} +} + +func (c *ProjectColl) GetCollectionName() string { + return c.coll +} + +func (c *ProjectColl) List() ([]*models.Project, error) { + var resp []*models.Project + + cursor, err := c.Collection.Find(context.TODO(), bson.M{}) + if err != nil { + if err == mongo.ErrNoDocuments { + return nil, nil + } + return nil, err + } + err = cursor.All(context.TODO(), &resp) + if err != nil { + return nil, err + } + + return resp, nil +} diff --git a/pkg/cli/upgradeassistant/internal/repository/mongodb/project_cluster_relation.go b/pkg/cli/upgradeassistant/internal/repository/mongodb/project_cluster_relation.go new file mode 100644 index 0000000000000000000000000000000000000000..a9b0e393165c87b4041cd5def65cecf9f848847e --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/mongodb/project_cluster_relation.go @@ -0,0 +1,59 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "errors" + "time" + + "go.mongodb.org/mongo-driver/mongo" + + models "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/config" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type ProjectClusterRelationColl struct { + *mongo.Collection + + coll string +} + +func NewProjectClusterRelationColl() *ProjectClusterRelationColl { + name := models.ProjectClusterRelation{}.TableName() + return &ProjectClusterRelationColl{ + Collection: mongotool.Database(config.MongoDatabase()).Collection(name), + coll: name, + } +} + +func (c *ProjectClusterRelationColl) GetCollectionName() string { + return c.coll +} + +func (c *ProjectClusterRelationColl) Create(args *models.ProjectClusterRelation) error { + if args == nil { + return errors.New("nil projectClusterRelation info") + } + + args.CreatedAt = time.Now().Unix() + args.CreatedBy = "system" + _, err := c.InsertOne(context.TODO(), args) + + return err +} diff --git a/pkg/cli/upgradeassistant/internal/repository/mongodb/registry_namespace.go b/pkg/cli/upgradeassistant/internal/repository/mongodb/registry_namespace.go new file mode 100644 index 0000000000000000000000000000000000000000..e814edd29c544b786ab090bb9ae465cceb0787b7 --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/mongodb/registry_namespace.go @@ -0,0 +1,140 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type FindRegOps struct { + ID string `json:"id"` + RegAddr string `json:"reg_addr"` + RegType string `json:"reg_type"` + RegProvider string `json:"reg_provider"` + IsDefault bool `json:"is_default"` + Namespace string `json:"namespace"` +} + +type RegistryNamespaceColl struct { + *mongo.Collection + + coll string +} + +func NewRegistryNamespaceColl() *RegistryNamespaceColl { + name := models.RegistryNamespace{}.TableName() + coll := &RegistryNamespaceColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} + + return coll +} + +func (r *RegistryNamespaceColl) GetCollectionName() string { + return r.coll +} + +func (r *RegistryNamespaceColl) EnsureIndex(ctx context.Context) error { + mod := mongo.IndexModel{ + Keys: bson.D{ + bson.E{Key: "org_id", Value: 1}, + bson.E{Key: "is_default", Value: 1}, + bson.E{Key: "reg_type", Value: 1}, + }, + Options: options.Index().SetUnique(false), + } + + _, err := r.Indexes().CreateOne(ctx, mod) + return err +} + +func (opt FindRegOps) getQuery() bson.M { + query := bson.M{} + + if opt.ID != "" { + oid, err := primitive.ObjectIDFromHex(opt.ID) + if err == nil { + query["_id"] = oid + } + } + + if opt.IsDefault { + query["is_default"] = true + } + + if opt.RegType != "" { + query["reg_type"] = opt.RegType + } + + if opt.RegAddr != "" { + query["reg_addr"] = opt.RegAddr + } + + if opt.RegProvider != "" { + query["reg_provider"] = opt.RegProvider + } + + if opt.Namespace != "" { + query["namespace"] = opt.Namespace + } + return query +} + +func (r *RegistryNamespaceColl) Find(opt *FindRegOps) (*models.RegistryNamespace, error) { + query := opt.getQuery() + + res := &models.RegistryNamespace{} + err := r.FindOne(context.TODO(), query).Decode(res) + if err != nil { + if err == mongo.ErrNoDocuments { + return nil, nil + } + return nil, err + } + + return res, nil +} + +func (r *RegistryNamespaceColl) FindAll(opt *FindRegOps) ([]*models.RegistryNamespace, error) { + query := opt.getQuery() + + ctx := context.Background() + opts := options.Find() + resp := make([]*models.RegistryNamespace, 0) + + cursor, err := r.Collection.Find(ctx, query, opts) + if err != nil { + if err == mongo.ErrNoDocuments { + return nil, nil + } + return nil, err + } + + err = cursor.All(ctx, &resp) + if err != nil { + return nil, err + } + + return resp, err +} diff --git a/pkg/cli/upgradeassistant/internal/repository/mongodb/testing.go b/pkg/cli/upgradeassistant/internal/repository/mongodb/testing.go new file mode 100644 index 0000000000000000000000000000000000000000..ee420afdf160ca9d4cad382a378c961ce3edbcee --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/mongodb/testing.go @@ -0,0 +1,66 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/config" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type ListTestOption struct { +} + +type TestingColl struct { + *mongo.Collection + + coll string +} + +func NewTestingColl() *TestingColl { + name := models.Testing{}.TableName() + return &TestingColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} +} + +func (c *TestingColl) GetCollectionName() string { + return c.coll +} + +func (c *TestingColl) List(opt *ListTestOption) ([]*models.Testing, error) { + query := bson.M{} + + var resp []*models.Testing + ctx := context.Background() + opts := options.Find() + cursor, err := c.Collection.Find(ctx, query, opts) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &resp) + if err != nil { + return nil, err + } + + return resp, nil +} diff --git a/pkg/cli/upgradeassistant/internal/repository/mongodb/workflow.go b/pkg/cli/upgradeassistant/internal/repository/mongodb/workflow.go new file mode 100644 index 0000000000000000000000000000000000000000..04c810bfdcf28e44e823194a11cc33759b9ed1a6 --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/mongodb/workflow.go @@ -0,0 +1,91 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/config" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type ListWorkflowOption struct { + IsSort bool + Projects []string + Names []string + Ids []string +} + +type WorkflowColl struct { + *mongo.Collection + + coll string +} + +func NewWorkflowColl() *WorkflowColl { + name := models.Workflow{}.TableName() + return &WorkflowColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} +} + +func (c *WorkflowColl) GetCollectionName() string { + return c.coll +} + +func (c *WorkflowColl) List(opt *ListWorkflowOption) ([]*models.Workflow, error) { + query := bson.M{} + if len(opt.Projects) > 0 { + query["product_tmpl_name"] = bson.M{"$in": opt.Projects} + } + if len(opt.Names) > 0 { + query["name"] = bson.M{"$in": opt.Names} + } + if len(opt.Ids) > 0 { + var oids []primitive.ObjectID + for _, id := range opt.Ids { + oid, err := primitive.ObjectIDFromHex(id) + if err != nil { + return nil, err + } + oids = append(oids, oid) + } + query["_id"] = bson.M{"$in": oids} + } + + resp := make([]*models.Workflow, 0) + ctx := context.Background() + opts := options.Find() + if opt.IsSort { + opts.SetSort(bson.D{{"update_time", -1}}) + } + cursor, err := c.Collection.Find(ctx, query, opts) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &resp) + if err != nil { + return nil, err + } + + return resp, nil +} diff --git a/pkg/cli/upgradeassistant/internal/repository/orm/addmysql.sql b/pkg/cli/upgradeassistant/internal/repository/orm/addmysql.sql new file mode 100644 index 0000000000000000000000000000000000000000..ec2ad6da6b695e8aaa376eca1f6d1129637b2606 --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/orm/addmysql.sql @@ -0,0 +1,3 @@ +use `%s`; + +ALTER TABLE `user` ADD `api_token` varchar(1024) NOT NULL DEFAULT '' COMMENT 'openAPIToken'; \ No newline at end of file diff --git a/pkg/cli/upgradeassistant/internal/repository/orm/db.go b/pkg/cli/upgradeassistant/internal/repository/orm/db.go new file mode 100644 index 0000000000000000000000000000000000000000..81b60c3432b27f67db4766a07147a89cd47e890c --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/orm/db.go @@ -0,0 +1,64 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package orm + +import ( + "database/sql" + _ "embed" + "fmt" + + _ "github.com/go-sql-driver/mysql" + + "github.com/koderover/zadig/pkg/config" +) + +type DbEditAction string + +const ( + DbEditActionAdd DbEditAction = "add" + DbEditActionDrop DbEditAction = "drop" +) + +//go:embed addmysql.sql +var addmysql []byte + +//go:embed dropmysql.sql +var dropmysql []byte + +func UpdateUserDBTables(action DbEditAction) error { + var mysql string + switch action { + case DbEditActionAdd: + mysql = fmt.Sprintf(string(addmysql), config.MysqlUserDB()) + + case DbEditActionDrop: + mysql = fmt.Sprintf(string(dropmysql), config.MysqlUserDB()) + } + if len(mysql) == 0 { + return fmt.Errorf("%smysql.sql is empty", action) + } + db, err := sql.Open("mysql", fmt.Sprintf( + "%s:%s@tcp(%s)/?charset=utf8&multiStatements=true", + config.MysqlUser(), config.MysqlPassword(), config.MysqlHost(), + )) + if err != nil { + return err + } + defer db.Close() + _, err = db.Exec(mysql) + return err +} diff --git a/pkg/cli/upgradeassistant/internal/repository/orm/dropmysql.sql b/pkg/cli/upgradeassistant/internal/repository/orm/dropmysql.sql new file mode 100644 index 0000000000000000000000000000000000000000..9d0db576c10064496aa363784103daa2a268fa93 --- /dev/null +++ b/pkg/cli/upgradeassistant/internal/repository/orm/dropmysql.sql @@ -0,0 +1,5 @@ +use `%s`; + +LOCK TABLES `user` WRITE; + ALTER TABLE `user` DROP COLUMN `api_token`; + UNLOCK TABLES; \ No newline at end of file diff --git a/pkg/cli/upgradeassistant/internal/upgradepath/dag.go b/pkg/cli/upgradeassistant/internal/upgradepath/dag.go index 3a98764c8515147ec89ebe356e790d708925d169..0319ded52fe336789c7cca8d499bea91cfc61c53 100644 --- a/pkg/cli/upgradeassistant/internal/upgradepath/dag.go +++ b/pkg/cli/upgradeassistant/internal/upgradepath/dag.go @@ -31,6 +31,22 @@ type upgradePath struct { var dag = dijkstra.NewGraph() var handlerMap = make(map[upgradePath]handler) +type RegisteredHandler struct { + FromVersion string + ToVersion string + Fn handler +} + +var RegisteredHandlers = make([]*RegisteredHandler, 0) + +func RegisterHandler(fromVersion, toVersion string, fn handler) { + RegisteredHandlers = append(RegisteredHandlers, &RegisteredHandler{ + FromVersion: fromVersion, + ToVersion: toVersion, + Fn: fn, + }) +} + func AddHandler(from, to int, fn handler) { if v, err := dag.GetVertex(from); err != nil || v.ID != from { dag.AddVertex(from) @@ -46,7 +62,7 @@ func AddHandler(from, to int, fn handler) { } func UpgradeWithBestPath(from, to string) error { - return upgradeWithBestPath(versionMap.From(from), versionMap.To(to)) + return upgradeWithBestPath(VersionDatas.VersionIndex(from), VersionDatas.VersionIndex(to)) } func upgradeWithBestPath(from, to int) error { diff --git a/pkg/cli/upgradeassistant/internal/upgradepath/versions.go b/pkg/cli/upgradeassistant/internal/upgradepath/versions.go index b57300a7571a71bda15e3f876e35310c1afceb90..ec960cac048c197752da0c842e13f371da89423b 100644 --- a/pkg/cli/upgradeassistant/internal/upgradepath/versions.go +++ b/pkg/cli/upgradeassistant/internal/upgradepath/versions.go @@ -17,90 +17,29 @@ limitations under the License. package upgradepath import ( - "sort" - "github.com/blang/semver/v4" + "github.com/koderover/zadig/pkg/tool/log" ) -const ( - Latest = iota + 1 - V130 - V131 - V140 - V150 -) - -var versionMap versions = map[string]int{ - "1.3.0": V130, - "1.3.1": V131, - "1.4.0": V140, - "1.5.0": V150, -} - -type versions map[string]int - -func (v versions) From(version string) int { - f, err := semver.Make(version) - if err != nil { - return Latest - } - res, ok := v[f.FinalizeVersion()] - if !ok { - return Latest - } - - return res -} +var VersionDatas versionList -func (v versions) To(version string) int { - t, err := semver.Make(version) - if err != nil { - return Latest - } - res, ok := v[t.FinalizeVersion()] - if !ok { - return Latest - } +type versionList semver.Versions - return res -} - -func (v versions) Max() int { - return v[v.MaxVersionString()] -} - -func (v versions) MaxVersionString() string { - max, _ := semver.Make("0.0.1") - for ver := range v { - sv, _ := semver.Make(ver) - if sv.GT(max) { - max = sv +func (v versionList) VersionIndex(version string) int { + for index, _version := range VersionDatas { + if _version.String() == version { + return index } } - - return max.FinalizeVersion() -} - -func init() { - versionList := make([]string, 0, len(versionMap)) - for v := range versionMap { - versionList = append(versionList, v) - } - - sort.Strings(versionList) - - for i := 0; i < len(versionList)-2; i++ { - lowVersion := versionList[i] - highVersion := versionList[i+1] - AddHandler(versionMap[lowVersion], versionMap[highVersion], defaultUpgradeHandler) - AddHandler(versionMap[highVersion], versionMap[lowVersion], defaultRollBackHandler) - } + return 0 } -func defaultUpgradeHandler() error { +func DefaultUpgradeHandler() error { + log.Info("default upgrade handler ") return nil } -func defaultRollBackHandler() error { +func DefaultRollBackHandler() error { + log.Info("default rollback handler ") return nil } diff --git a/pkg/config/config.go b/pkg/config/config.go index 69855b8350ba8aff08a4a779c052e30b10b0f749..ff97af75ec9e15610c895bc7bddb322716c83b5b 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -73,10 +73,6 @@ func RequestLogFile() string { return LogPath() + RequestLogName() } -func PoetryAPIRootKey() string { - return viper.GetString(setting.ENVPoetryAPIRootKey) -} - func GetServiceByCode(code int) *setting.ServiceInfo { return setting.Services[code] } @@ -85,6 +81,10 @@ func AslanServiceInfo() *setting.ServiceInfo { return GetServiceByCode(setting.Aslan) } +func SecretKey() string { + return viper.GetString(setting.ENVSecretKey) +} + func AslanServiceAddress() string { s := AslanServiceInfo() return GetServiceAddress(s.Name, s.Port) @@ -142,12 +142,12 @@ func CollieServiceAddress() string { return GetServiceAddress(s.Name, s.Port) } -func PoetryServiceInfo() *setting.ServiceInfo { - return GetServiceByCode(setting.Poetry) +func ConfigServiceInfo() *setting.ServiceInfo { + return GetServiceByCode(setting.Config) } -func PoetryServiceAddress() string { - s := PoetryServiceInfo() +func ConfigServiceAddress() string { + s := ConfigServiceInfo() return GetServiceAddress(s.Name, s.Port) } @@ -168,6 +168,33 @@ func OPAServiceAddress() string { return GetServiceAddress(s.Name, s.Port) } +func PolicyServiceInfo() *setting.ServiceInfo { + return GetServiceByCode(setting.Policy) +} + +func PolicyServiceAddress() string { + s := PolicyServiceInfo() + return GetServiceAddress(s.Name, s.Port) +} + +func UserServiceInfo() *setting.ServiceInfo { + return GetServiceByCode(setting.User) +} + +func UserServiceAddress() string { + s := UserServiceInfo() + return GetServiceAddress(s.Name, s.Port) +} + +func VendorServiceInfo() *setting.ServiceInfo { + return GetServiceByCode(setting.Vendor) +} + +func VendorServiceAddress() string { + s := VendorServiceInfo() + return GetServiceAddress(s.Name, s.Port) +} + func GetServiceAddress(name string, port int32) string { return fmt.Sprintf("http://%s:%d", name, port) } @@ -192,6 +219,10 @@ func ObjectStorageTemplatePath(name, kind string) string { return filepath.Join("templates", kind, name) } +func ObjectStorageDeliveryVersionPath(project string) string { + return filepath.Join("delivery-distributes", "files", project) +} + func ObjectStorageChartTemplatePath(name string) string { return ObjectStorageTemplatePath(name, setting.ChartTemplatesPath) } @@ -200,6 +231,10 @@ func LocalServicePath(project, service string) string { return filepath.Join(DataPath(), project, service) } +func LocalServicePathWithRevision(project, service, revision string) string { + return filepath.Join(DataPath(), project, service, revision) +} + func LocalTemplatePath(name, kind string) string { return filepath.Join(DataPath(), "templates", kind, name) } @@ -215,3 +250,39 @@ func MongoURI() string { func MongoDatabase() string { return viper.GetString(setting.ENVAslanDBName) } + +func MysqlUser() string { + return viper.GetString(setting.ENVMysqlUser) +} + +func MysqlUserDB() string { + return viper.GetString(setting.ENVMysqlUserDB) +} + +func MysqlPassword() string { + return viper.GetString(setting.ENVMysqlPassword) +} + +func MysqlHost() string { + return viper.GetString(setting.ENVMysqlHost) +} + +func AdminEmail() string { + return viper.GetString(setting.ENVAdminEmail) +} + +func AdminPassword() string { + return viper.GetString(setting.ENVAdminPassword) +} + +func Namespace() string { + return viper.GetString(setting.ENVNamespace) +} + +func RoleBindingNameFromUIDAndRole(uid string, role setting.RoleType, roleNamespace string) string { + return fmt.Sprintf("%s-%s-%s", uid, role, roleNamespace) +} + +func BuildResourceKey(resourceType, projectName, labelBinding string) string { + return fmt.Sprintf("%s-%s-%s", resourceType, projectName, labelBinding) +} diff --git a/pkg/handler/cache/cache.go b/pkg/handler/cache/cache.go new file mode 100644 index 0000000000000000000000000000000000000000..82cd80523ef343208db53763f913ed1d15a1398b --- /dev/null +++ b/pkg/handler/cache/cache.go @@ -0,0 +1,65 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cache + +import ( + "fmt" + "sync" +) + +type cacheStore struct { + store *sync.Map +} + +func NewCache() Cacher { + return &cacheStore{ + store: &sync.Map{}, + } +} + +func (c *cacheStore) Get(key string) (value interface{}, exists bool) { + return c.store.Load(key) +} + +func (c *cacheStore) Set(key string, value interface{}) { + c.store.Store(key, value) +} + +func (c *cacheStore) Delete(key string) { + c.store.Delete(key) +} + +func (c *cacheStore) List() map[string]interface{} { + data := map[string]interface{}{} + c.store.Range(func(key, value interface{}) bool { + data[fmt.Sprint(key)] = value + + return true + }) + + return data +} + +func (c *cacheStore) Purge() { + // Note: + // Don't use `c.store = &sync.Map{}` because it's dangerous when there's a race condition on `c.store.` + c.store.Range(func(key, value interface{}) bool { + c.store.Delete(key) + + return true + }) +} diff --git a/pkg/handler/cache/consts.go b/pkg/handler/cache/consts.go new file mode 100644 index 0000000000000000000000000000000000000000..98a8f2c3fbb2969d82767d41b89266bdc68be847 --- /dev/null +++ b/pkg/handler/cache/consts.go @@ -0,0 +1,21 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cache + +const MsgSuccess = "success" + +const HeaderXData = "x-data" diff --git a/pkg/handler/cache/handler.go b/pkg/handler/cache/handler.go new file mode 100644 index 0000000000000000000000000000000000000000..cfc2ac8f3929452b6b4de827b4a87fb157e262af --- /dev/null +++ b/pkg/handler/cache/handler.go @@ -0,0 +1,84 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cache + +import ( + "fmt" + + "github.com/gin-gonic/gin" + + internalhandler "github.com/koderover/zadig/pkg/shared/handler" +) + +type handlers struct { + cache Cacher +} + +func NewHandlers() *handlers { + return &handlers{ + cache: NewCache(), + } +} + +func (h *handlers) Get(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + data, _ := h.cache.Get(c.Param("key")) + if data == nil { + data = "" + } + + c.Header(HeaderXData, fmt.Sprint(data)) + ctx.Resp, ctx.Err = data, nil +} + +func (h *handlers) Set(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + h.cache.Set(c.Param("key"), c.Param("value")) + + ctx.Resp, ctx.Err = MsgSuccess, nil +} + +func (h *handlers) Delete(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + h.cache.Delete(c.Param("key")) + + ctx.Resp, ctx.Err = MsgSuccess, nil +} + +func (h *handlers) List(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + cacheData := h.cache.List() + + ctx.Resp, ctx.Err = cacheData, nil +} + +func (h *handlers) Purge(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + h.cache.Purge() + + ctx.Resp, ctx.Err = MsgSuccess, nil +} diff --git a/pkg/handler/cache/interface.go b/pkg/handler/cache/interface.go new file mode 100644 index 0000000000000000000000000000000000000000..ab2abede2bf442c1c47fb01dcef1b39e3706c8b3 --- /dev/null +++ b/pkg/handler/cache/interface.go @@ -0,0 +1,53 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cache + +import "github.com/gin-gonic/gin" + +type Cacher interface { + // Get gets data based on `key`. + Get(key string) (value interface{}, exists bool) + + // Set sets data `key:values`. + Set(key string, value interface{}) + + // Delete deletes data based on `key`. + Delete(key string) + + // List lists all of data. + List() map[string]interface{} + + // Purge purges all of data. + Purge() +} + +type CacheHandler interface { + // Get is a gin handler that gets cache data. + Get(c *gin.Context) + + // Set is a gin handler that sets cache data. + Set(c *gin.Context) + + // Delete is a gin handler that deletes cache data. + Delete(c *gin.Context) + + // List is a gin handler that lists cache data. + List(c *gin.Context) + + // Purge is a gin handler that purges cache data. + Purge(c *gin.Context) +} diff --git a/pkg/handler/cache/router.go b/pkg/handler/cache/router.go new file mode 100644 index 0000000000000000000000000000000000000000..e83bad039914b955b72a68da4465c3c2ea4b7951 --- /dev/null +++ b/pkg/handler/cache/router.go @@ -0,0 +1,39 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cache + +import "github.com/gin-gonic/gin" + +type Router struct { + handlers CacheHandler +} + +func NewRouter() *Router { + return &Router{ + handlers: NewHandlers(), + } +} + +func (r *Router) Inject(router *gin.RouterGroup) { + router.GET("/:key", r.handlers.Get) + router.GET("/", r.handlers.List) + + router.POST("/:key/:value", r.handlers.Set) + + router.DELETE("/:key", r.handlers.Delete) + router.DELETE("/", r.handlers.Purge) +} diff --git a/pkg/microservice/aslan/config/config.go b/pkg/microservice/aslan/config/config.go index d27203b6352cdfe2552df1bbc34da8707666eae9..c2b14a77822e6c5ff5231562063f662f3c9e1966 100644 --- a/pkg/microservice/aslan/config/config.go +++ b/pkg/microservice/aslan/config/config.go @@ -74,14 +74,6 @@ func LogLevel() int { return viper.GetInt(setting.ENVLogLevel) } -func PoetryAPIServer() string { - return configbase.PoetryServiceAddress() -} - -func PoetryAPIRootKey() string { - return viper.GetString(setting.ENVPoetryAPIRootKey) -} - func CollieAPIAddress() string { return configbase.CollieServiceAddress() } @@ -106,6 +98,10 @@ func HubAgentImage() string { return viper.GetString(setting.ENVHubAgentImage) } +func ResourceServerImage() string { + return viper.GetString(setting.ENVResourceServerImage) +} + func KodespaceVersion() string { return viper.GetString(setting.ENVKodespaceVersion) } @@ -198,6 +194,10 @@ func PredatorImage() string { return viper.GetString(setting.ENVPredatorImage) } +func PackagerImage() string { + return viper.GetString(setting.EnvPackagerImage) +} + func DockerHosts() []string { return strings.Split(viper.GetString(setting.ENVDockerHosts), ",") } @@ -231,5 +231,25 @@ func ObjectStorageServicePath(project, service string) string { } func LocalServicePath(project, service string) string { - return configbase.LocalServicePath(project, service) + return configbase.LocalServicePathWithRevision(project, service, "latest") +} + +func LocalServicePathWithRevision(project, service string, revision int64) string { + return configbase.LocalServicePathWithRevision(project, service, fmt.Sprintf("%d", revision)) +} + +func LocalDeliveryChartPathWithRevision(project, service string, revision int64) string { + return configbase.LocalServicePathWithRevision(project, service, fmt.Sprintf("delivery/%d", revision)) +} + +func ServiceNameWithRevision(serviceName string, revision int64) string { + return fmt.Sprintf("%s-%d", serviceName, revision) +} + +func ServiceAccountNameForUser(userID string) string { + return fmt.Sprintf("%s-sa", userID) +} + +func DindImage() string { + return viper.GetString(setting.DindImage) } diff --git a/pkg/microservice/aslan/config/consts.go b/pkg/microservice/aslan/config/consts.go index f6a1b9df424629e4dbe7e379fc18a09e4e8415b5..b708a13b2b5cd380327f0a212f47e5abb4f55a19 100644 --- a/pkg/microservice/aslan/config/consts.go +++ b/pkg/microservice/aslan/config/consts.go @@ -18,12 +18,15 @@ package config import ( "regexp" + + "github.com/koderover/zadig/pkg/setting" ) var ( // RenderTemplateAlias ... RenderTemplateAlias = regexp.MustCompile(`{{\s?\.\w+\s?}}`) ServiceNameAlias = regexp.MustCompile(`\$Service\$`) + ProductNameAlias = regexp.MustCompile(`\$Product\$`) NameSpaceRegex = regexp.MustCompile(NameSpaceRegexString) ) @@ -81,6 +84,12 @@ const ( TestType PipelineType = "test" // ServiceType 服务 ServiceType PipelineType = "service" + // WorkflowTypeV3 + WorkflowTypeV3 PipelineType = "workflow_v3" + // ArtifactPackageType package artifact + ArtifactType PipelineType = "artifact" + // ScanningType is the type for scanning + ScanningType PipelineType = "scanning" ) type Status string @@ -98,6 +107,10 @@ const ( StatusQueued Status = "queued" StatusBlocked Status = "blocked" QueueItemPending Status = "pending" + StatusChanged Status = "changed" + StatusNotRun Status = "notRun" + StatusPrepare Status = "prepare" + StatusReject Status = "reject" ) type TaskStatus string @@ -115,33 +128,81 @@ const ( type TaskType string const ( - TaskPipeline TaskType = "pipeline" - TaskBuild TaskType = "buildv2" - TaskJenkinsBuild TaskType = "jenkins_build" - TaskArtifact TaskType = "artifact" - TaskDeploy TaskType = "deploy" - TaskTestingV2 TaskType = "testingv2" - TaskDistributeToS3 TaskType = "distribute2kodo" - TaskReleaseImage TaskType = "release_image" - TaskJira TaskType = "jira" - TaskDockerBuild TaskType = "docker_build" - TaskSecurity TaskType = "security" - TaskResetImage TaskType = "reset_image" - TaskDistribute TaskType = "distribute" + TaskPipeline TaskType = "pipeline" + TaskBuild TaskType = "buildv2" + TaskBuildV3 TaskType = "buildv3" + TaskJenkinsBuild TaskType = "jenkins_build" + TaskArtifact TaskType = "artifact" + TaskArtifactDeploy TaskType = "artifact_deploy" + TaskDeploy TaskType = "deploy" + TaskTestingV2 TaskType = "testingv2" + TaskScanning TaskType = "scanning" + TaskDistributeToS3 TaskType = "distribute2kodo" + TaskReleaseImage TaskType = "release_image" + TaskJira TaskType = "jira" + TaskDockerBuild TaskType = "docker_build" + TaskSecurity TaskType = "security" + TaskResetImage TaskType = "reset_image" + TaskDistribute TaskType = "distribute" + TaskTrigger TaskType = "trigger" + TaskExtension TaskType = "extension" + TaskArtifactPackage TaskType = "artifact_package" ) -type DistributeType string +type StepType string const ( - File DistributeType = "file" - Image DistributeType = "image" + StepTools StepType = "tools" + StepShell StepType = "shell" + StepGit StepType = "git" + StepDockerBuild StepType = "docker_build" + StepDeploy StepType = "deploy" + StepHelmDeploy StepType = "helm_deploy" + StepImageDistribute StepType = "image_distribute" + StepArchive StepType = "archive" + StepArchiveDistribute StepType = "archive_distribute" + StepJunitReport StepType = "junit_report" + StepHtmlReport StepType = "html_report" +) + +type JobType string + +const ( + JobBuild JobType = "build" + JobDeploy JobType = "deploy" + JobZadigBuild JobType = "zadig-build" + JobZadigDeploy JobType = "zadig-deploy" + JobFreestyle JobType = "freestyle" + JobPlugin JobType = "plugin" +) + +type ApproveOrReject string + +const ( + Approve ApproveOrReject = "approve" + Reject ApproveOrReject = "reject" +) + +type DeploySourceType string + +const ( + SourceRuntime DeploySourceType = "runtime" + SourceFromJob DeploySourceType = "fromjob" +) + +type StageType string + +const ( + StageCustom StageType = "custom" + StepApproval StageType = "approval" ) -type K8SClusterStatus string +type DistributeType string const ( - Disconnected K8SClusterStatus = "disconnected" - Pending K8SClusterStatus = "pending" + File DistributeType = "file" + Image DistributeType = "image" + Chart DistributeType = "chart" ) type NotifyType int @@ -157,20 +218,6 @@ const ( NameSpaceRegexString = "[^a-z0-9.-]" ) -// Request ... -type Request string - -const ( - // HighRequest 16 CPU 32 G - HighRequest = Request("high") - // MediumRequest 8 CPU 16 G - MediumRequest = Request("medium") - // LowRequest 4 CPU 8 G - LowRequest = Request("low") - // MinRequest 2 CPU 2 G - MinRequest = Request("min") -) - //ProductPermission ... type ProductPermission string @@ -196,6 +243,7 @@ type HookEventType string const ( HookEventPush = HookEventType("push") HookEventPr = HookEventType("pull_request") + HookEventTag = HookEventType("tag") HookEventUpdated = HookEventType("ref-updated") ) @@ -212,10 +260,25 @@ const ( ) const ( - SWRProvider = "swr" + RegistryTypeSWR = "swr" + RegistryTypeAWS = "ecr" ) const ( ImageResourceType = "image" TarResourceType = "tar" ) + +const ( + RoleBindingNameEdit = setting.ProductName + "-edit" + RoleBindingNameView = setting.ProductName + "-view" +) + +type CommonEnvCfgType string + +const ( + CommonEnvCfgTypeIngress CommonEnvCfgType = "Ingress" + CommonEnvCfgTypeConfigMap CommonEnvCfgType = "ConfigMap" + CommonEnvCfgTypeSecret CommonEnvCfgType = "Secret" + CommonEnvCfgTypePvc CommonEnvCfgType = "PVC" +) diff --git a/pkg/microservice/aslan/core/build/handler/build.go b/pkg/microservice/aslan/core/build/handler/build.go index 5cf11416c0ad0f5f7f700de46858d69b20a433c1..7dae2b09a56c98101be0c095e8220d111fd16004 100644 --- a/pkg/microservice/aslan/core/build/handler/build.go +++ b/pkg/microservice/aslan/core/build/handler/build.go @@ -34,14 +34,25 @@ func FindBuildModule(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = buildservice.FindBuild(c.Param("name"), c.Query("productName"), ctx.Logger) + ctx.Resp, ctx.Err = buildservice.FindBuild(c.Param("name"), c.Query("projectName"), ctx.Logger) } func ListBuildModules(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = buildservice.ListBuild(c.Query("name"), c.Query("targets"), c.Query("productName"), ctx.Logger) + ctx.Resp, ctx.Err = buildservice.ListBuild(c.Query("name"), c.Query("targets"), c.Query("projectName"), ctx.Logger) +} + +func ListBuildModulesByServiceModule(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + var excludeJenkins bool + if c.Query("excludeJenkins") == "true" { + excludeJenkins = true + } + + ctx.Resp, ctx.Err = buildservice.ListBuildModulesByServiceModule(c.Query("encryptedKey"), c.Query("projectName"), excludeJenkins, ctx.Logger) } func CreateBuildModule(c *gin.Context) { @@ -56,7 +67,7 @@ func CreateBuildModule(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("CreateBuildModule json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, args.ProductName, "新增", "项目管理-构建", args.Name, string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, args.ProductName, "新增", "项目管理-构建", args.Name, string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) err = c.BindJSON(args) @@ -65,7 +76,7 @@ func CreateBuildModule(c *gin.Context) { return } - ctx.Err = buildservice.CreateBuild(ctx.Username, args, ctx.Logger) + ctx.Err = buildservice.CreateBuild(ctx.UserName, args, ctx.Logger) } func UpdateBuildModule(c *gin.Context) { @@ -80,7 +91,7 @@ func UpdateBuildModule(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("UpdateBuildModule json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, args.ProductName, "更新", "项目管理-构建", args.Name, string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, args.ProductName, "更新", "项目管理-构建", args.Name, string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) err = c.BindJSON(args) @@ -88,7 +99,7 @@ func UpdateBuildModule(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid Build args") return } - ctx.Err = buildservice.UpdateBuild(ctx.Username, args, ctx.Logger) + ctx.Err = buildservice.UpdateBuild(ctx.UserName, args, ctx.Logger) } func DeleteBuildModule(c *gin.Context) { @@ -96,8 +107,8 @@ func DeleteBuildModule(c *gin.Context) { defer func() { internalhandler.JSONResponse(c, ctx) }() name := c.Query("name") - productName := c.Query("productName") - internalhandler.InsertOperationLog(c, ctx.Username, productName, "删除", "项目管理-构建", name, "", ctx.Logger) + productName := c.Query("projectName") + internalhandler.InsertOperationLog(c, ctx.UserName, productName, "删除", "项目管理-构建", name, "", ctx.Logger) if name == "" { ctx.Err = e.ErrInvalidParam.AddDesc("empty Name") @@ -126,7 +137,7 @@ func UpdateBuildTargets(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddErr(err) return } - internalhandler.InsertOperationLog(c, ctx.Username, c.Query("productName"), "更新", "项目管理-服务组件", args.Name, string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, c.Query("projectName"), "更新", "项目管理-服务组件", args.Name, string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) if err := c.BindJSON(args); err != nil { @@ -134,5 +145,5 @@ func UpdateBuildTargets(c *gin.Context) { return } - ctx.Err = buildservice.UpdateBuildTargets(args.Name, c.Query("productName"), args.Targets, ctx.Logger) + ctx.Err = buildservice.UpdateBuildTargets(args.Name, c.Query("projectName"), args.Targets, ctx.Logger) } diff --git a/pkg/microservice/aslan/core/build/handler/router.go b/pkg/microservice/aslan/core/build/handler/router.go index f55bd25c153ed46c6b6f6789cfe745cd9f4d56ec..f52c21542e7e45cbb17f4265eb30f17d59d49c27 100644 --- a/pkg/microservice/aslan/core/build/handler/router.go +++ b/pkg/microservice/aslan/core/build/handler/router.go @@ -18,24 +18,20 @@ package handler import ( "github.com/gin-gonic/gin" - - gin2 "github.com/koderover/zadig/pkg/middleware/gin" - "github.com/koderover/zadig/pkg/types/permission" ) type Router struct{} func (*Router) Inject(router *gin.RouterGroup) { - router.Use(gin2.Auth()) - build := router.Group("build") { build.GET("/:name", FindBuildModule) build.GET("", ListBuildModules) - build.POST("", gin2.StoreProductName, gin2.IsHavePermission([]string{permission.BuildManageUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, CreateBuildModule) - build.PUT("", gin2.StoreProductName, gin2.IsHavePermission([]string{permission.BuildManageUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, UpdateBuildModule) - build.DELETE("", gin2.IsHavePermission([]string{permission.BuildDeleteUUID}, permission.QueryType), gin2.UpdateOperationLogStatus, DeleteBuildModule) - build.POST("/targets", gin2.IsHavePermission([]string{permission.BuildManageUUID}, permission.QueryType), gin2.UpdateOperationLogStatus, UpdateBuildTargets) + build.GET("/serviceModule", ListBuildModulesByServiceModule) + build.POST("", CreateBuildModule) + build.PUT("", UpdateBuildModule) + build.DELETE("", DeleteBuildModule) + build.POST("/targets", UpdateBuildTargets) } target := router.Group("targets") diff --git a/pkg/microservice/aslan/core/build/handler/target.go b/pkg/microservice/aslan/core/build/handler/target.go index 71f5e160084ebee3b9e02d3bf2e7112c3d6baf19..2978c4020390a1f126ffea7f8d8cbd888846a4f4 100644 --- a/pkg/microservice/aslan/core/build/handler/target.go +++ b/pkg/microservice/aslan/core/build/handler/target.go @@ -27,7 +27,7 @@ func ListDeployTarget(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = buildservice.ListDeployTarget(c.Query("productName"), ctx.Logger) + ctx.Resp, ctx.Err = buildservice.ListDeployTarget(c.Query("projectName"), ctx.Logger) } func ListBuildModulesForProduct(c *gin.Context) { diff --git a/pkg/microservice/aslan/core/build/service/build.go b/pkg/microservice/aslan/core/build/service/build.go index 186ca028072fcfbd79b67debb97c718b3d5baa6c..602933b9035e314b2c3b336847d1c02d52c469e5 100644 --- a/pkg/microservice/aslan/core/build/service/build.go +++ b/pkg/microservice/aslan/core/build/service/build.go @@ -28,20 +28,32 @@ import ( commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + commonutil "github.com/koderover/zadig/pkg/microservice/aslan/core/common/util" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types" ) type BuildResp struct { ID string `json:"id"` Name string `json:"name"` Targets []*commonmodels.ServiceModuleTarget `json:"targets"` + KeyVals []*commonmodels.KeyVal `json:"key_vals"` + Repos []*types.Repository `json:"repos"` UpdateTime int64 `json:"update_time"` UpdateBy string `json:"update_by"` Pipelines []string `json:"pipelines"` ProductName string `json:"productName"` } +type ServiceModuleAndBuildResp struct { + ServiceName string `json:"service_name"` + ServiceModule string `json:"service_module"` + ModuleBuilds []*BuildResp `json:"module_builds"` +} + func FindBuild(name, productName string, log *zap.SugaredLogger) (*commonmodels.Build, error) { opt := &commonrepo.BuildFindOption{ Name: name, @@ -107,14 +119,109 @@ func ListBuild(name, targets, productName string, log *zap.SugaredLogger) ([]*Bu return resp, nil } -func CreateBuild(username string, build *commonmodels.Build, log *zap.SugaredLogger) error { +func ListBuildModulesByServiceModule(encryptedKey, productName string, excludeJenkins bool, log *zap.SugaredLogger) ([]*ServiceModuleAndBuildResp, error) { + services, err := commonrepo.NewServiceColl().ListMaxRevisionsByProduct(productName) + if err != nil { + return nil, e.ErrListBuildModule.AddErr(err) + } + + serviceModuleAndBuildResp := make([]*ServiceModuleAndBuildResp, 0) + for _, serviceTmpl := range services { + for _, container := range serviceTmpl.Containers { + opt := &commonrepo.BuildListOption{ + ServiceName: serviceTmpl.ServiceName, + Targets: []string{container.Name}, + } + if serviceTmpl.Visibility != setting.PublicService { + opt.ProductName = productName + } + + buildModules, err := commonrepo.NewBuildColl().List(opt) + if err != nil { + return nil, e.ErrListBuildModule.AddErr(err) + } + var resp []*BuildResp + for _, build := range buildModules { + if excludeJenkins && build.JenkinsBuild != nil { + continue + } + // get build env vars when it's a template build + if build.TemplateID != "" { + templateEnvs := []*commonmodels.KeyVal{} + buildTemplate, err := commonrepo.NewBuildTemplateColl().Find(&commonrepo.BuildTemplateQueryOption{ + ID: build.TemplateID, + }) + // if template not found, envs are empty, but do not block user. + if err != nil { + log.Errorf("build job: %s, template not found", build.Name) + } else { + templateEnvs = buildTemplate.PreBuild.Envs + } + + for _, target := range build.Targets { + if target.ServiceModule == container.Name && target.ServiceName == serviceTmpl.ServiceName { + build.PreBuild.Envs = target.Envs + build.Repos = target.Repos + } + } + build.PreBuild.Envs = commonservice.MergeBuildEnvs(templateEnvs, build.PreBuild.Envs) + } + if err := commonservice.EncryptKeyVals(encryptedKey, build.PreBuild.Envs, log); err != nil { + return serviceModuleAndBuildResp, err + } + resp = append(resp, &BuildResp{ + ID: build.ID.Hex(), + Name: build.Name, + KeyVals: build.PreBuild.Envs, + Repos: build.Repos, + }) + } + serviceModuleAndBuildResp = append(serviceModuleAndBuildResp, &ServiceModuleAndBuildResp{ + ServiceName: serviceTmpl.ServiceName, + ServiceModule: container.Name, + ModuleBuilds: resp, + }) + } + } + return serviceModuleAndBuildResp, nil +} +func fillBuildTargetData(build *commonmodels.Build) error { + if build.TemplateID == "" { + return nil + } + buildTemplate, err := commonrepo.NewBuildTemplateColl().Find(&commonrepo.BuildTemplateQueryOption{ + ID: build.TemplateID, + }) + if err != nil { + return fmt.Errorf("failed to find build template with id: %s, err: %s", build.TemplateID, err) + } + build.Targets = make([]*commonmodels.ServiceModuleTarget, 0, len(build.TargetRepos)) + for _, target := range build.TargetRepos { + build.Targets = append(build.Targets, &commonmodels.ServiceModuleTarget{ + ProductName: target.Service.ProductName, + ServiceName: target.Service.ServiceName, + ServiceModule: target.Service.ServiceModule, + Repos: target.Repos, + Envs: commonservice.MergeBuildEnvs(buildTemplate.PreBuild.Envs, target.Envs), + }) + } + return nil +} + +func CreateBuild(username string, build *commonmodels.Build, log *zap.SugaredLogger) error { if len(build.Name) == 0 { return e.ErrCreateBuildModule.AddDesc("empty name") } + if err := commonutil.CheckDefineResourceParam(build.PreBuild.ResReq, build.PreBuild.ResReqSpec); err != nil { + return e.ErrCreateBuildModule.AddDesc(err.Error()) + } build.UpdateBy = username - correctFields(build) + err := correctFields(build) + if err != nil { + return err + } if err := commonrepo.NewBuildColl().Create(build); err != nil { log.Errorf("[Build.Upsert] %s error: %v", build.Name, err) @@ -128,16 +235,26 @@ func UpdateBuild(username string, build *commonmodels.Build, log *zap.SugaredLog if len(build.Name) == 0 { return e.ErrUpdateBuildModule.AddDesc("empty name") } + if err := commonutil.CheckDefineResourceParam(build.PreBuild.ResReq, build.PreBuild.ResReqSpec); err != nil { + return e.ErrUpdateBuildModule.AddDesc(err.Error()) + } existed, err := commonrepo.NewBuildColl().Find(&commonrepo.BuildFindOption{Name: build.Name, ProductName: build.ProductName}) if err == nil && existed.PreBuild != nil && build.PreBuild != nil { commonservice.EnsureSecretEnvs(existed.PreBuild.Envs, build.PreBuild.Envs) } - correctFields(build) + err = correctFields(build) + if err != nil { + return err + } + + if err = updateCvmService(build, existed); err != nil { + log.Warnf("failed to update cvm service,err:%s", err) + } + build.UpdateBy = username build.UpdateTime = time.Now().Unix() - if err := commonrepo.NewBuildColl().Update(build); err != nil { log.Errorf("[Build.Upsert] %s error: %v", build.Name, err) return e.ErrUpdateBuildModule.AddErr(err) @@ -146,8 +263,54 @@ func UpdateBuild(username string, build *commonmodels.Build, log *zap.SugaredLog return nil } -func DeleteBuild(name, productName string, log *zap.SugaredLogger) error { +func updateCvmService(currentBuild, oldBuild *commonmodels.Build) error { + deleteServices := sets.NewString() + currentServiceModuleKey := sets.NewString() + for _, currentServiceModule := range currentBuild.Targets { + currentServiceModuleKey.Insert(fmt.Sprintf("%s-%s-%s", currentServiceModule.ProductName, currentServiceModule.ServiceName, currentServiceModule.ServiceModule)) + } + + for _, oldServiceModule := range oldBuild.Targets { + if !currentServiceModuleKey.Has(fmt.Sprintf("%s-%s-%s", oldServiceModule.ProductName, oldServiceModule.ServiceName, oldServiceModule.ServiceModule)) { + deleteServices.Insert(oldServiceModule.ServiceName) + } + } + + for _, serviceName := range deleteServices.List() { + opt := &commonrepo.ServiceFindOption{ + ServiceName: serviceName, + Type: setting.PMDeployType, + ProductName: currentBuild.ProductName, + ExcludeStatus: setting.ProductStatusDeleting, + } + + resp, err := commonrepo.NewServiceColl().Find(opt) + if err != nil { + continue + } + serviceTemplate := fmt.Sprintf(setting.ServiceTemplateCounterName, resp.ServiceName, resp.ProductName) + rev, err := commonrepo.NewCounterColl().GetNextSeq(serviceTemplate) + if err != nil { + return err + } + resp.Revision = rev + + if err := commonrepo.NewServiceColl().Delete(resp.ServiceName, resp.Type, resp.ProductName, setting.ProductStatusDeleting, resp.Revision); err != nil { + log.Errorf("failed to delete service %s, error: %s", resp.ServiceName, err) + return err + } + resp.BuildName = "" + if err := commonrepo.NewServiceColl().Create(resp); err != nil { + log.Errorf("failed to delete service %s, error: %s", resp.ServiceName, err) + return err + } + } + + return nil +} + +func DeleteBuild(name, productName string, log *zap.SugaredLogger) error { if len(name) == 0 { return e.ErrDeleteBuildModule.AddDesc("empty name") } @@ -313,7 +476,11 @@ func UpdateBuildTargets(name, productName string, targets []*commonmodels.Servic return nil } -func correctFields(build *commonmodels.Build) { +func correctFields(build *commonmodels.Build) error { + err := fillBuildTargetData(build) + if err != nil { + return err + } // make sure cache has no empty field caches := make([]string, 0) for _, cache := range build.Caches { @@ -329,6 +496,42 @@ func correctFields(build *commonmodels.Build) { build.PostBuild.DockerBuild.DockerFile = strings.Trim(build.PostBuild.DockerBuild.DockerFile, " ") build.PostBuild.DockerBuild.WorkDir = strings.Trim(build.PostBuild.DockerBuild.WorkDir, " ") } + if build.TemplateID == "" { + for _, repo := range build.Repos { + if repo.Source != setting.SourceFromOther { + continue + } + modifyAuthType(repo) + } + return nil + } + + for _, target := range build.Targets { + for _, repo := range target.Repos { + if repo.Source != setting.SourceFromOther { + continue + } + modifyAuthType(repo) + } + } + return nil +} + +func modifyAuthType(repo *types.Repository) { + repo.RepoOwner = strings.TrimPrefix(repo.RepoOwner, "/") + repo.RepoOwner = strings.TrimSuffix(repo.RepoOwner, "/") + repo.RepoName = strings.TrimPrefix(repo.RepoName, "/") + repo.RepoName = strings.TrimSuffix(repo.RepoName, "/") + codehosts, err := systemconfig.New().ListCodeHostsInternal() + if err != nil { + log.Errorf("failed to list codehost,err:%s", err) + } + for _, codehost := range codehosts { + if repo.CodehostID == codehost.ID { + repo.AuthType = codehost.AuthType + break + } + } } func verifyBuildTargets(name, productName string, targets []*commonmodels.ServiceModuleTarget, log *zap.SugaredLogger) error { diff --git a/pkg/microservice/aslan/core/build/service/target.go b/pkg/microservice/aslan/core/build/service/target.go index 016c0ee7ce927962d6bb0a6f90cf208ac0c0df2c..28052d26d70a7aa06333e9cc14d2ed6f7d087505 100644 --- a/pkg/microservice/aslan/core/build/service/target.go +++ b/pkg/microservice/aslan/core/build/service/target.go @@ -20,7 +20,6 @@ import ( "fmt" "go.uber.org/zap" - "k8s.io/apimachinery/pkg/util/sets" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" @@ -37,39 +36,22 @@ func ListDeployTarget(productName string, log *zap.SugaredLogger) ([]*commonmode return serviceObjects, e.ErrListTemplate.AddDesc(errMsg) } - //获取该项目下的的build的targets - buildTargets := sets.NewString() - builds, err := commonrepo.NewBuildColl().List(&commonrepo.BuildListOption{ProductName: productName}) - if err != nil { - log.Errorf("[Build.List] %s error: %v", productName, err) - return nil, e.ErrListBuildModule.AddErr(err) - } - for _, build := range builds { - for _, serviceModuleTarget := range build.Targets { - buildTargets.Insert(fmt.Sprintf("%s-%s", serviceModuleTarget.ServiceName, serviceModuleTarget.ServiceModule)) - } - } - for _, svc := range services { switch svc.Type { case setting.K8SDeployType, setting.HelmDeployType: for _, container := range svc.Containers { - if !buildTargets.Has(fmt.Sprintf("%s-%s", svc.ServiceName, container.Name)) { - serviceObjects = append(serviceObjects, &commonmodels.ServiceModuleTarget{ - ProductName: svc.ProductName, - ServiceName: svc.ServiceName, - ServiceModule: container.Name, - }) - } - } - case setting.PMDeployType: - if !buildTargets.Has(fmt.Sprintf("%s-%s", svc.ServiceName, svc.ServiceName)) { serviceObjects = append(serviceObjects, &commonmodels.ServiceModuleTarget{ ProductName: svc.ProductName, ServiceName: svc.ServiceName, - ServiceModule: svc.ServiceName, + ServiceModule: container.Name, }) } + case setting.PMDeployType: + serviceObjects = append(serviceObjects, &commonmodels.ServiceModuleTarget{ + ProductName: svc.ProductName, + ServiceName: svc.ServiceName, + ServiceModule: svc.ServiceName, + }) } } diff --git a/pkg/microservice/aslan/core/code/client/codehub/codehub.go b/pkg/microservice/aslan/core/code/client/codehub/codehub.go new file mode 100644 index 0000000000000000000000000000000000000000..008b42d41cef55b2d11620aa079bcd121efcf230 --- /dev/null +++ b/pkg/microservice/aslan/core/code/client/codehub/codehub.go @@ -0,0 +1,114 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package codehub + +import ( + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/code/client" + "github.com/koderover/zadig/pkg/tool/codehub" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +type Config struct { + Region string `json:"region"` + AccessKey string `json:"application_id"` + SecretKey string `json:"client_secret"` + EnableProxy bool `json:"enable_proxy"` +} + +type Client struct { + Client *codehub.CodeHubClient +} + +func (c *Config) Open(id int, logger *zap.SugaredLogger) (client.CodeHostClient, error) { + codehubClient := codehub.NewCodeHubClient(c.AccessKey, c.SecretKey, c.Region, config.ProxyHTTPSAddr(), c.EnableProxy) + return &Client{Client: codehubClient}, nil +} + +func (c *Client) ListBranches(opt client.ListOpt) ([]*client.Branch, error) { + bList, err := c.Client.BranchList(opt.ProjectName) + if err != nil { + return nil, err + } + var res []*client.Branch + for _, o := range bList { + res = append(res, &client.Branch{ + Name: o.Name, + Protected: o.Protected, + Merged: o.Merged, + }) + } + return res, nil +} + +func (c *Client) ListTags(opt client.ListOpt) ([]*client.Tag, error) { + tags, err := c.Client.TagList(opt.ProjectName) + if err != nil { + return nil, err + } + + var res []*client.Tag + for _, o := range tags { + res = append(res, &client.Tag{ + Name: o.Name, + }) + } + + return res, nil +} + +func (c *Client) ListPrs(opt client.ListOpt) ([]*client.PullRequest, error) { + return nil, nil +} + +func (c *Client) ListNamespaces(keyword string) ([]*client.Namespace, error) { + nsList, err := c.Client.NamespaceList() + if err != nil { + return nil, err + } + var res []*client.Namespace + for _, o := range nsList { + res = append(res, &client.Namespace{ + Name: o.Name, + Path: o.Path, + Kind: o.Kind, + ProjectUUID: o.ProjectUUID, + }) + } + return res, nil +} + +func (c *Client) ListProjects(opt client.ListOpt) ([]*client.Project, error) { + projects, err := c.Client.RepoList(opt.Namespace, opt.Key, opt.PerPage) + if err != nil { + return nil, e.ErrCodehostListProjects.AddDesc(err.Error()) + } + var res []*client.Project + for _, project := range projects { + res = append(res, &client.Project{ + Name: project.Name, + Description: project.Description, + DefaultBranch: project.DefaultBranch, + Namespace: project.Namespace, + RepoUUID: project.RepoUUID, + RepoID: project.RepoID, + }) + } + return res, nil +} diff --git a/pkg/microservice/aslan/core/code/client/gerrit/gerrit.go b/pkg/microservice/aslan/core/code/client/gerrit/gerrit.go new file mode 100644 index 0000000000000000000000000000000000000000..723643b8cc9cc4f5cd75301788517640b06191fc --- /dev/null +++ b/pkg/microservice/aslan/core/code/client/gerrit/gerrit.go @@ -0,0 +1,102 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gerrit + +import ( + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/code/client" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/gerrit" + gerrittool "github.com/koderover/zadig/pkg/tool/gerrit" + + "go.uber.org/zap" +) + +type Config struct { + Address string `json:"address"` + AccessToken string `json:"access_token"` + AccessKey string `json:"application_id"` + EnableProxy bool `json:"enable_proxy"` +} + +type Client struct { + Client *gerrit.Client +} + +func (c *Config) Open(id int, logger *zap.SugaredLogger) (client.CodeHostClient, error) { + gerritClient := gerrit.NewClient(c.Address, c.AccessToken, config.ProxyHTTPSAddr(), c.EnableProxy) + return &Client{Client: gerritClient}, nil +} + +func (c *Client) ListBranches(opt client.ListOpt) ([]*client.Branch, error) { + bList, err := c.Client.ListBranches(opt.ProjectName) + if err != nil { + return nil, err + } + var res []*client.Branch + for _, o := range bList { + res = append(res, &client.Branch{ + Name: o, + }) + } + return res, nil +} + +func (c *Client) ListTags(opt client.ListOpt) ([]*client.Tag, error) { + result, err := c.Client.ListTags(opt.ProjectName) + if err != nil { + return nil, err + } + var res []*client.Tag + for _, o := range result { + res = append(res, &client.Tag{ + Name: o.Ref, + Message: o.Message, + }) + } + return res, nil +} + +func (c *Client) ListPrs(opt client.ListOpt) ([]*client.PullRequest, error) { + return nil, nil +} + +func (c *Client) ListNamespaces(keyword string) ([]*client.Namespace, error) { + return []*client.Namespace{{ + Name: gerrit.DefaultNamespace, + Path: gerrit.DefaultNamespace, + Kind: client.OrgKind, + }}, nil +} + +func (c *Client) ListProjects(opt client.ListOpt) ([]*client.Project, error) { + projects, err := c.Client.ListProjectsByKey(opt.Key) + if err != nil { + return nil, e.ErrCodehostListProjects.AddDesc(err.Error()) + } + var res []*client.Project + for ind, o := range projects { + res = append(res, &client.Project{ + ID: ind, // fake id + Name: gerrittool.Unescape(o.ID), // id could have %2F + Description: o.Description, + DefaultBranch: "master", + Namespace: gerrittool.DefaultNamespace, + }) + } + return res, nil +} diff --git a/pkg/microservice/aslan/core/code/client/gitee/gitee.go b/pkg/microservice/aslan/core/code/client/gitee/gitee.go new file mode 100644 index 0000000000000000000000000000000000000000..6e5b99b24ca4d5b253640b61cd1482469f31ea11 --- /dev/null +++ b/pkg/microservice/aslan/core/code/client/gitee/gitee.go @@ -0,0 +1,157 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gitee + +import ( + "context" + + giteeClient "gitee.com/openeuler/go-gitee/gitee" + "github.com/antihax/optional" + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/code/client" + "github.com/koderover/zadig/pkg/tool/gitee" +) + +type Config struct { + AccessToken string `json:"access_token"` + EnableProxy bool `json:"enable_proxy"` +} + +type Client struct { + Client *gitee.Client + AccessToken string +} + +func (c *Config) Open(id int, logger *zap.SugaredLogger) (client.CodeHostClient, error) { + client := gitee.NewClient(id, c.AccessToken, config.ProxyHTTPSAddr(), c.EnableProxy) + return &Client{ + Client: client, + AccessToken: c.AccessToken, + }, nil +} + +func (c *Client) ListBranches(opt client.ListOpt) ([]*client.Branch, error) { + bList, err := c.Client.ListBranches(context.TODO(), opt.Namespace, opt.ProjectName, nil) + if err != nil { + return nil, err + } + var res []*client.Branch + for _, o := range bList { + res = append(res, &client.Branch{ + Name: o.Name, + Protected: o.Protected, + }) + } + return res, nil +} + +func (c *Client) ListTags(opt client.ListOpt) ([]*client.Tag, error) { + tags, err := c.Client.ListTags(context.TODO(), c.AccessToken, opt.Namespace, opt.ProjectName) + if err != nil { + return nil, err + } + var resp []*client.Tag + for _, tag := range tags { + resp = append(resp, &client.Tag{ + Name: tag.Name, + Message: tag.Message, + }) + } + + return resp, nil +} + +func (c *Client) ListPrs(opt client.ListOpt) ([]*client.PullRequest, error) { + prs, err := c.Client.ListPullRequests(context.TODO(), opt.Namespace, opt.ProjectName, &giteeClient.GetV5ReposOwnerRepoPullsOpts{ + PerPage: optional.NewInt32(100), + }) + if err != nil { + return nil, err + } + var res []*client.PullRequest + for _, o := range prs { + res = append(res, &client.PullRequest{ + ID: int(o.Number), + State: o.State, + User: o.User.Login, + Number: int(o.Number), + AuthorUsername: o.User.Login, + Title: o.Title, + SourceBranch: o.Base.Ref, + TargetBranch: o.Base.Ref, + }) + } + return res, nil +} + +func (c *Client) ListNamespaces(keyword string) ([]*client.Namespace, error) { + user, err := c.Client.GetAuthenticatedUser(context.TODO()) + if err != nil { + return nil, err + } + namespaceUser := client.Namespace{ + Name: user.Login, + Path: user.Login, + Kind: client.UserKind, + } + + organizations, err := c.Client.ListOrganizationsForAuthenticatedUser(context.TODO()) + if err != nil { + return nil, err + } + + var res []*client.Namespace + res = append(res, &namespaceUser) + for _, o := range organizations { + res = append(res, &client.Namespace{ + Name: o.Login, + Path: o.Login, + Kind: client.OrgKind, + }) + } + return res, nil +} + +func (c *Client) ListProjects(opt client.ListOpt) ([]*client.Project, error) { + var projects []gitee.Project + var err error + switch opt.NamespaceType { + case client.OrgKind: + projects, err = c.Client.ListRepositoriesForOrg(c.AccessToken, opt.Namespace, opt.Page, opt.PerPage) + if err != nil { + return nil, err + } + default: + projects, err = c.Client.ListRepositoriesForAuthenticatedUser(c.AccessToken, opt.Key, opt.Page, opt.PerPage) + if err != nil { + return nil, err + } + } + + var res []*client.Project + for _, project := range projects { + res = append(res, &client.Project{ + ID: project.ID, + Name: project.Name, + DefaultBranch: project.DefaultBranch, + }) + } + + return res, nil +} diff --git a/pkg/microservice/aslan/core/code/client/github/github.go b/pkg/microservice/aslan/core/code/client/github/github.go new file mode 100644 index 0000000000000000000000000000000000000000..53854fafd436930ec0173d6f4815dd675ba8a1d1 --- /dev/null +++ b/pkg/microservice/aslan/core/code/client/github/github.go @@ -0,0 +1,156 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package github + +import ( + "context" + + github2 "github.com/google/go-github/v35/github" + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/code/client" + "github.com/koderover/zadig/pkg/tool/git/github" +) + +type Config struct { + AccessToken string `json:"access_token"` + EnableProxy bool `json:"enable_proxy"` +} + +type Client struct { + Client *github.Client +} + +func (c *Config) Open(id int, logger *zap.SugaredLogger) (client.CodeHostClient, error) { + + cfg := &github.Config{ + AccessToken: c.AccessToken, + } + if c.EnableProxy { + cfg.Proxy = config.ProxyHTTPSAddr() + } + return &Client{ + Client: github.NewClient(cfg), + }, nil +} + +func (c *Client) ListBranches(opt client.ListOpt) ([]*client.Branch, error) { + bList, err := c.Client.ListBranches(context.TODO(), opt.Namespace, opt.ProjectName, nil) + if err != nil { + return nil, err + } + var res []*client.Branch + for _, o := range bList { + res = append(res, &client.Branch{ + Name: o.GetName(), + Protected: o.GetProtected(), + }) + } + return res, nil +} + +func (c *Client) ListTags(opt client.ListOpt) ([]*client.Tag, error) { + tags, err := c.Client.ListTags(context.TODO(), opt.Namespace, opt.ProjectName, nil) + if err != nil { + return nil, err + } + var res []*client.Tag + for _, o := range tags { + res = append(res, &client.Tag{ + Name: o.GetName(), + ZipballURL: o.GetZipballURL(), + TarballURL: o.GetTarballURL(), + }) + } + return res, nil +} + +func (c *Client) ListPrs(opt client.ListOpt) ([]*client.PullRequest, error) { + prs, err := c.Client.ListPullRequests(context.TODO(), opt.Namespace, opt.ProjectName, &github2.PullRequestListOptions{ + ListOptions: github2.ListOptions{PerPage: 100}, + }) + if err != nil { + return nil, err + } + var res []*client.PullRequest + for _, o := range prs { + res = append(res, &client.PullRequest{ + ID: o.GetNumber(), + CreatedAt: o.GetCreatedAt().Unix(), + UpdatedAt: o.GetUpdatedAt().Unix(), + State: o.GetState(), + User: o.GetUser().GetLogin(), + Number: o.GetNumber(), + AuthorUsername: o.GetUser().GetLogin(), + Title: o.GetTitle(), + SourceBranch: o.GetHead().GetRef(), + TargetBranch: o.GetBase().GetRef(), + }) + } + return res, nil +} + +func (c *Client) ListNamespaces(keyword string) ([]*client.Namespace, error) { + user, err := c.Client.GetAuthenticatedUser(context.TODO()) + if err != nil { + return nil, err + } + namespaceUser := client.Namespace{ + Name: user.GetLogin(), + Path: user.GetLogin(), + Kind: client.UserKind, + } + + organizations, err := c.Client.ListOrganizationsForAuthenticatedUser(context.TODO(), nil) + if err != nil { + return nil, err + } + + var res []*client.Namespace + res = append(res, &namespaceUser) + for _, o := range organizations { + res = append(res, &client.Namespace{ + Name: o.GetLogin(), + Path: o.GetLogin(), + Kind: client.OrgKind, + }) + } + return res, nil +} + +func (c *Client) ListProjects(opt client.ListOpt) ([]*client.Project, error) { + repos, err := c.Client.ListRepositoriesForAuthenticatedUser(context.TODO(), opt.Namespace, nil) + if err != nil { + return nil, err + } + var res []*client.Project + for _, o := range repos { + // Note. when using gitHub api to filter repos, private repo will not be returned + // we need to filter the repos with query namespace when repo list is returned + if len(opt.Namespace) > 0 && o.GetOwner().GetLogin() != opt.Namespace { + continue + } + res = append(res, &client.Project{ + ID: int(o.GetID()), + Name: o.GetName(), + DefaultBranch: o.GetDefaultBranch(), + Namespace: o.GetOwner().GetLogin(), + }) + } + return res, nil +} diff --git a/pkg/microservice/aslan/core/code/client/gitlab/gitlab.go b/pkg/microservice/aslan/core/code/client/gitlab/gitlab.go new file mode 100644 index 0000000000000000000000000000000000000000..46a7cc0eee3c51752d76e0880df94b2c5d9c2c5a --- /dev/null +++ b/pkg/microservice/aslan/core/code/client/gitlab/gitlab.go @@ -0,0 +1,166 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gitlab + +import ( + "go.uber.org/zap" + + gogitlab "github.com/xanzy/go-gitlab" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/code/client" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/git/gitlab" +) + +type Config struct { + Address string `json:"address"` + AccessToken string `json:"access_token"` + // the field determine whether the proxy is enabled + EnableProxy bool `json:"enable_proxy"` +} + +type Client struct { + Client *gitlab.Client +} + +func (c *Config) Open(id int, logger *zap.SugaredLogger) (client.CodeHostClient, error) { + + client, err := gitlab.NewClient(id, c.Address, c.AccessToken, config.ProxyHTTPSAddr(), c.EnableProxy) + if err != nil { + return nil, err + } + return &Client{Client: client}, nil +} + +func (c *Client) ListBranches(opt client.ListOpt) ([]*client.Branch, error) { + bList, err := c.Client.ListBranches(opt.Namespace, opt.ProjectName, opt.Key, &gitlab.ListOptions{ + Page: opt.Page, + PerPage: opt.PerPage, + NoPaginated: true, + }) + if err != nil { + return nil, err + } + var res []*client.Branch + for _, b := range bList { + res = append(res, &client.Branch{ + Name: b.Name, + Protected: b.Protected, + Merged: b.Merged, + }) + } + return res, nil +} + +func (c *Client) ListTags(opt client.ListOpt) ([]*client.Tag, error) { + tags, err := c.Client.ListTags(opt.Namespace, opt.ProjectName, &gitlab.ListOptions{ + Page: opt.Page, + PerPage: opt.PerPage, + NoPaginated: true, + }, opt.Key) + if err != nil { + return nil, err + } + var res []*client.Tag + for _, o := range tags { + res = append(res, &client.Tag{ + Name: o.Name, + Message: o.Message, + }) + } + return res, nil +} + +func (c *Client) ListPrs(opt client.ListOpt) ([]*client.PullRequest, error) { + prs, err := c.Client.ListOpenedProjectMergeRequests(opt.Namespace, opt.ProjectName, opt.TargeBr, opt.Key, &gitlab.ListOptions{ + Page: opt.Page, + PerPage: opt.PerPage, + NoPaginated: true, + }) + if err != nil { + return nil, err + } + var res []*client.PullRequest + for _, o := range prs { + res = append(res, &client.PullRequest{ + ID: o.IID, + TargetBranch: o.TargetBranch, + SourceBranch: o.SourceBranch, + ProjectID: o.ProjectID, + Title: o.Title, + State: o.State, + CreatedAt: o.CreatedAt.Unix(), + UpdatedAt: o.UpdatedAt.Unix(), + AuthorUsername: o.Author.Username, + }) + } + return res, nil +} + +func (c *Client) ListNamespaces(keyword string) ([]*client.Namespace, error) { + nsList, err := c.Client.ListNamespaces(keyword, nil) + if err != nil { + return nil, err + } + var res []*client.Namespace + for _, o := range nsList { + res = append(res, &client.Namespace{ + Name: o.Path, + Path: o.FullPath, + Kind: o.Kind, + }) + } + return res, nil +} + +func (c *Client) ListProjects(opt client.ListOpt) ([]*client.Project, error) { + var projects []*gogitlab.Project + var err error + switch opt.NamespaceType { + case client.GroupKind: + projects, err = c.Client.ListGroupProjects(opt.Namespace, opt.Key, &gitlab.ListOptions{ + Page: opt.Page, + PerPage: opt.PerPage, + NoPaginated: true, + }) + if err != nil { + return nil, e.ErrCodehostListProjects.AddDesc(err.Error()) + } + default: + projects, err = c.Client.ListUserProjects(opt.Namespace, opt.Key, &gitlab.ListOptions{ + Page: opt.Page, + PerPage: opt.PerPage, + NoPaginated: true, + }) + if err != nil { + return nil, e.ErrCodehostListProjects.AddDesc(err.Error()) + } + + } + var res []*client.Project + for _, o := range projects { + res = append(res, &client.Project{ + ID: o.ID, + Name: o.Path, + Namespace: o.Namespace.FullPath, + Description: o.Description, + DefaultBranch: o.DefaultBranch, + }) + } + return res, nil +} diff --git a/pkg/microservice/aslan/core/code/client/interface.go b/pkg/microservice/aslan/core/code/client/interface.go new file mode 100644 index 0000000000000000000000000000000000000000..783633059f8308c9fc113020c9b5d1483cd9336f --- /dev/null +++ b/pkg/microservice/aslan/core/code/client/interface.go @@ -0,0 +1,86 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +const ( + OrgKind = "org" + GroupKind = "group" + UserKind = "user" +) + +type CodeHostClient interface { + ListBranches(opt ListOpt) ([]*Branch, error) + ListTags(opt ListOpt) ([]*Tag, error) + ListPrs(opt ListOpt) ([]*PullRequest, error) + ListNamespaces(keyword string) ([]*Namespace, error) + ListProjects(opt ListOpt) ([]*Project, error) +} + +type ListOpt struct { + Namespace string + NamespaceType string + ProjectName string + Key string + Page int + PerPage int + TargeBr string +} + +type Branch struct { + Name string `json:"name"` + Protected bool `json:"protected"` + Merged bool `json:"merged"` +} + +type Tag struct { + Name string `json:"name"` + ZipballURL string `json:"zipball_url"` + TarballURL string `json:"tarball_url"` + Message string `json:"message"` +} + +type PullRequest struct { + ID int `json:"id"` + TargetBranch string `json:"targetBranch"` + SourceBranch string `json:"sourceBranch"` + ProjectID int `json:"projectId"` + Title string `json:"title"` + State string `json:"state"` + CreatedAt int64 `json:"createdAt"` + UpdatedAt int64 `json:"updatedAt"` + AuthorUsername string `json:"authorUsername"` + Number int `json:"number"` + User string `json:"user"` + Base string `json:"base,omitempty"` +} + +type Namespace struct { + Name string `json:"name"` + Path string `json:"path"` + Kind string `json:"kind"` + ProjectUUID string `json:"project_uuid,omitempty"` +} + +type Project struct { + ID int `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + DefaultBranch string `json:"defaultBranch"` + Namespace string `json:"namespace"` + RepoUUID string `json:"repo_uuid,omitempty"` + RepoID string `json:"repo_id,omitempty"` +} diff --git a/pkg/microservice/aslan/core/code/client/open/open.go b/pkg/microservice/aslan/core/code/client/open/open.go new file mode 100644 index 0000000000000000000000000000000000000000..2e9ae976befe80aa33878406ce38dfed559a788e --- /dev/null +++ b/pkg/microservice/aslan/core/code/client/open/open.go @@ -0,0 +1,64 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package open + +import ( + "encoding/json" + "fmt" + + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/code/client" + "github.com/koderover/zadig/pkg/microservice/aslan/core/code/client/codehub" + "github.com/koderover/zadig/pkg/microservice/aslan/core/code/client/gerrit" + "github.com/koderover/zadig/pkg/microservice/aslan/core/code/client/gitee" + "github.com/koderover/zadig/pkg/microservice/aslan/core/code/client/github" + "github.com/koderover/zadig/pkg/microservice/aslan/core/code/client/gitlab" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" +) + +type ClientConfig interface { + Open(id int, logger *zap.SugaredLogger) (client.CodeHostClient, error) +} + +var ClientsConfig = map[string]func() ClientConfig{ + setting.SourceFromGitlab: func() ClientConfig { return new(gitlab.Config) }, + setting.SourceFromGithub: func() ClientConfig { return new(github.Config) }, + setting.SourceFromGerrit: func() ClientConfig { return new(gerrit.Config) }, + setting.SourceFromCodeHub: func() ClientConfig { return new(codehub.Config) }, + setting.SourceFromGitee: func() ClientConfig { return new(gitee.Config) }, +} + +func OpenClient(ch *systemconfig.CodeHost, log *zap.SugaredLogger) (client.CodeHostClient, error) { + var c client.CodeHostClient + f, ok := ClientsConfig[ch.Type] + if !ok { + log.Error("unknow codehost type") + return c, fmt.Errorf("unknow codehost type") + } + clientConfig := f() + bs, err := json.Marshal(ch) + if err != nil { + return nil, err + } + if err := json.Unmarshal(bs, clientConfig); err != nil { + log.Errorf("marsh err:%s", err) + return nil, err + } + return clientConfig.Open(ch.ID, log) +} diff --git a/pkg/microservice/aslan/core/code/handler/codehost.go b/pkg/microservice/aslan/core/code/handler/codehost.go index 93b257bc330922e517126680df0fced0d75951c9..1b6e7e007cdbcaadb47575716f3aea624fcfc2ce 100644 --- a/pkg/microservice/aslan/core/code/handler/codehost.go +++ b/pkg/microservice/aslan/core/code/handler/codehost.go @@ -24,9 +24,8 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/core/code/service" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/codehost" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" internalhandler "github.com/koderover/zadig/pkg/shared/handler" - "github.com/koderover/zadig/pkg/shared/poetry" e "github.com/koderover/zadig/pkg/tool/errors" ) @@ -34,8 +33,8 @@ func GetCodeHostList(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - codeHostSlice := make([]*poetry.CodeHost, 0) - codeHosts, err := codehost.GetCodeHostList() + codeHostSlice := make([]*systemconfig.CodeHost, 0) + codeHosts, err := systemconfig.New().ListCodeHostsInternal() ctx.Err = err for _, codeHost := range codeHosts { codeHost.AccessToken = setting.MaskValue @@ -63,21 +62,26 @@ func CodeHostGetNamespaceList(c *gin.Context) { ctx.Resp, ctx.Err = service.CodeHostListNamespaces(chID, keyword, ctx.Logger) } +type CodeHostListProjectsArgs struct { + PerPage int `json:"per_page" form:"per_page,default=30"` + Page int `json:"page" form:"page,default=1"` + Key string `json:"key" form:"key"` +} + func CodeHostGetProjectsList(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() namespaceType := c.DefaultQuery("type", "group") codehostID := c.Param("codehostId") - namespace := c.Param("namespace") - keyword := c.Query("key") + repoOwner := c.Query("repoOwner") if codehostID == "" { ctx.Err = e.ErrInvalidParam.AddDesc("empty codehostId") return } - if namespace == "" { - ctx.Err = e.ErrInvalidParam.AddDesc("empty namespace") + if repoOwner == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("empty repoOwner") return } if namespaceType == "" { @@ -89,13 +93,38 @@ func CodeHostGetProjectsList(c *gin.Context) { return } + args := &CodeHostListProjectsArgs{} + if err := c.ShouldBindQuery(args); err != nil { + ctx.Err = err + return + } + chID, _ := strconv.Atoi(codehostID) - ctx.Resp, ctx.Err = service.CodeHostListProjects( + projects, err := service.CodeHostListProjects( chID, - strings.Replace(namespace, "%2F", "/", -1), + strings.Replace(repoOwner, "%2F", "/", -1), namespaceType, - keyword, + args.Page, + args.PerPage, + args.Key, ctx.Logger) + if err != nil { + ctx.Err = err + return + } + + for _, project := range projects { + if project.Namespace == "" { + project.Namespace = repoOwner + } + } + ctx.Resp = projects +} + +type CodeHostGetPageNateListArgs struct { + PerPage int `json:"per_page" form:"per_page,default=100"` + Page int `json:"page" form:"page,default=1"` + Key string `json:"key" form:"key"` } func CodeHostGetBranchList(c *gin.Context) { @@ -103,27 +132,35 @@ func CodeHostGetBranchList(c *gin.Context) { defer func() { internalhandler.JSONResponse(c, ctx) }() codehostID := c.Param("codehostId") - namespace := c.Param("namespace") - projectName := c.Param("projectName") // pro Name, id/name -> gitlab = id + repoOwner := c.Query("repoOwner") + repoName := c.Query("repoName") // pro Name, id/name -> gitlab = id + args := new(CodeHostGetPageNateListArgs) + if err := c.ShouldBindQuery(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } if codehostID == "" { ctx.Err = e.ErrInvalidParam.AddDesc("empty codehostId") return } - if namespace == "" { - ctx.Err = e.ErrInvalidParam.AddDesc("empty namespace") + if repoOwner == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("empty repoOwner") return } - if projectName == "" { - ctx.Err = e.ErrInvalidParam.AddDesc("empty projectName") + if repoName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("empty repoName") return } chID, _ := strconv.Atoi(codehostID) ctx.Resp, ctx.Err = service.CodeHostListBranches( chID, - projectName, - strings.Replace(namespace, "%2F", "/", -1), + repoName, + strings.Replace(repoOwner, "%2F", "/", -1), + args.Key, + args.Page, + args.PerPage, ctx.Logger) } @@ -132,24 +169,28 @@ func CodeHostGetTagList(c *gin.Context) { defer func() { internalhandler.JSONResponse(c, ctx) }() codehostID := c.Param("codehostId") - namespace := c.Param("namespace") - projectName := c.Param("projectName") // pro Name, id/name -> gitlab = id - + repoOwner := c.Query("repoOwner") + repoName := c.Query("repoName") // pro Name, id/name -> gitlab = id + args := new(CodeHostGetPageNateListArgs) + if err := c.ShouldBindQuery(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } if codehostID == "" { ctx.Err = e.ErrInvalidParam.AddDesc("empty codehostId") return } - if namespace == "" { - ctx.Err = e.ErrInvalidParam.AddDesc("empty namespace") + if repoOwner == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("empty repoOwner") return } - if projectName == "" { - ctx.Err = e.ErrInvalidParam.AddDesc("empty projectName") + if repoName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("empty repoName") return } chID, _ := strconv.Atoi(codehostID) - ctx.Resp, ctx.Err = service.CodeHostListTags(chID, projectName, strings.Replace(namespace, "%2F", "/", -1), ctx.Logger) + ctx.Resp, ctx.Err = service.CodeHostListTags(chID, repoName, strings.Replace(repoOwner, "%2F", "/", -1), args.Key, args.Page, args.PerPage, ctx.Logger) } func CodeHostGetPRList(c *gin.Context) { @@ -157,26 +198,32 @@ func CodeHostGetPRList(c *gin.Context) { defer func() { internalhandler.JSONResponse(c, ctx) }() codehostID := c.Param("codehostId") - namespace := c.Param("namespace") - projectName := c.Param("projectName") // pro Name, id/name -> gitlab = id + repoOwner := c.Query("repoOwner") + repoName := c.Query("repoName") // pro Name, id/name -> gitlab = id + + args := new(CodeHostGetPageNateListArgs) + if err := c.ShouldBindQuery(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } if codehostID == "" { ctx.Err = e.ErrInvalidParam.AddDesc("empty codehostId") return } - if namespace == "" { - ctx.Err = e.ErrInvalidParam.AddDesc("empty namespace") + if repoOwner == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("empty repoOwner") return } - if projectName == "" { - ctx.Err = e.ErrInvalidParam.AddDesc("empty projectName") + if repoName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("empty repoName") return } targetBr := c.Query("targetBranch") chID, _ := strconv.Atoi(codehostID) - ctx.Resp, ctx.Err = service.CodeHostListPRs(chID, projectName, strings.Replace(namespace, "%2F", "/", -1), targetBr, ctx.Logger) + ctx.Resp, ctx.Err = service.CodeHostListPRs(chID, repoName, strings.Replace(repoOwner, "%2F", "/", -1), targetBr, args.Key, args.Page, args.PerPage, ctx.Logger) } func ListRepoInfos(c *gin.Context) { @@ -189,6 +236,23 @@ func ListRepoInfos(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid repo args") return } - param := c.Query("param") - ctx.Resp, ctx.Err = service.ListRepoInfos(args.Infos, param, ctx.Logger) + ctx.Resp, ctx.Err = service.ListRepoInfos(args.Infos, ctx.Logger) +} + +type BranchesRequest struct { + Regular string `json:"regular"` + Branches []string `json:"branches"` +} + +func MatchBranchesList(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(BranchesRequest) + err := c.ShouldBindJSON(args) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid branches args") + return + } + ctx.Resp = service.MatchBranchesList(args.Regular, args.Branches) } diff --git a/pkg/microservice/aslan/core/code/handler/router.go b/pkg/microservice/aslan/core/code/handler/router.go index 6b0a22f8192f37ac48ae86257d39e9e7772e0d2e..123a9108512e263b468c5030a1e9f5b695a96c16 100644 --- a/pkg/microservice/aslan/core/code/handler/router.go +++ b/pkg/microservice/aslan/core/code/handler/router.go @@ -18,25 +18,21 @@ package handler import ( "github.com/gin-gonic/gin" - - gin2 "github.com/koderover/zadig/pkg/middleware/gin" - "github.com/koderover/zadig/pkg/types/permission" ) type Router struct{} func (*Router) Inject(router *gin.RouterGroup) { - router.Use(gin2.Auth()) - codehost := router.Group("codehost") { codehost.GET("", GetCodeHostList) codehost.GET("/:codehostId/namespaces", CodeHostGetNamespaceList) - codehost.GET("/:codehostId/namespaces/:namespace/projects", CodeHostGetProjectsList) - codehost.GET("/:codehostId/namespaces/:namespace/projects/:projectName/branches", CodeHostGetBranchList) - codehost.GET("/:codehostId/namespaces/:namespace/projects/:projectName/tags", CodeHostGetTagList) - codehost.GET("/:codehostId/namespaces/:namespace/projects/:projectName/prs", CodeHostGetPRList) + codehost.GET("/:codehostId/projects", CodeHostGetProjectsList) + codehost.GET("/:codehostId/branches", CodeHostGetBranchList) + codehost.GET("/:codehostId/tags", CodeHostGetTagList) + codehost.GET("/:codehostId/prs", CodeHostGetPRList) codehost.PUT("/infos", ListRepoInfos) + codehost.POST("/branches/regular/check", MatchBranchesList) } // --------------------------------------------------------------------------------------- @@ -44,11 +40,13 @@ func (*Router) Inject(router *gin.RouterGroup) { // --------------------------------------------------------------------------------------- workspace := router.Group("workspace") { - workspace.DELETE("", GetProductNameByWorkspacePipeline, gin2.IsHavePermission([]string{permission.WorkflowUpdateUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, CleanWorkspace) + workspace.DELETE("", GetProductNameByWorkspacePipeline, CleanWorkspace) workspace.GET("/file", GetWorkspaceFile) - workspace.GET("/git/:codehostId/:repoName/:branchName/:remoteName", GetGitRepoInfo) + workspace.GET("/git/:codehostId", GetGitRepoInfo) workspace.GET("/tree", GetRepoTree) - workspace.GET("/codehub/:codehostId/:repoUUID/:branchName", GetCodehubRepoInfo) + workspace.GET("/getcontents/:codehostId", GetContents) + workspace.GET("/codehub/:codehostId", GetCodehubRepoInfo) } + } diff --git a/pkg/microservice/aslan/core/code/handler/workspace.go b/pkg/microservice/aslan/core/code/handler/workspace.go index ae46b2e0835466375d9712bd52cc5d61b6b01b64..848339c29a5f9973dfe1bad00360cf5e79a7a8a0 100644 --- a/pkg/microservice/aslan/core/code/handler/workspace.go +++ b/pkg/microservice/aslan/core/code/handler/workspace.go @@ -17,7 +17,6 @@ limitations under the License. package handler import ( - "net/url" "strconv" "github.com/gin-gonic/gin" @@ -31,7 +30,7 @@ import ( func GetProductNameByWorkspacePipeline(c *gin.Context) { ctx := internalhandler.NewContext(c) - pipeline, err := commonservice.GetPipelineInfo(ctx.User.ID, c.Query("pipelineName"), ctx.Logger) + pipeline, err := commonservice.GetPipelineInfo(c.Query("pipelineName"), ctx.Logger) if err != nil { log.Errorf("GetProductNameByWorkspacePipeline err : %v", err) return @@ -43,13 +42,13 @@ func GetProductNameByWorkspacePipeline(c *gin.Context) { func CleanWorkspace(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - internalhandler.InsertOperationLog(c, ctx.Username, c.GetString("productName"), "清理", "单服务工作流-工作目录", c.Query("pipelineName"), "", ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, c.GetString("productName"), "清理", "单服务工作流-工作目录", c.Query("pipelineName"), "", ctx.Logger) name := c.Query("pipelineName") if name == "" { ctx.Err = e.ErrInvalidParam.AddDesc("empty pipeline name") return } - ctx.Err = service.CleanWorkspace(ctx.Username, name, ctx.Logger) + ctx.Err = service.CleanWorkspace(ctx.UserName, name, ctx.Logger) } func GetWorkspaceFile(c *gin.Context) { @@ -69,7 +68,7 @@ func GetWorkspaceFile(c *gin.Context) { return } - fileRealPath, err := service.GetWorkspaceFilePath(ctx.Username, name, file, ctx.Logger) + fileRealPath, err := service.GetWorkspaceFilePath(ctx.UserName, name, file, ctx.Logger) if err != nil { c.JSON(e.ErrorMessage(err)) c.Abort() @@ -94,24 +93,19 @@ func GetGitRepoInfo(c *gin.Context) { return } - repoName := c.Param("repoName") + repoName := c.Query("repoName") if repoName == "" { ctx.Err = e.ErrInvalidParam.AddDesc("empty repo name") return } - repoName, err = url.QueryUnescape(repoName) - if err != nil { - ctx.Err = e.ErrInvalidParam.AddDesc("repoName decode error") - return - } - branchName := c.Param("branchName") + branchName := c.Query("branchName") if branchName == "" { ctx.Err = e.ErrInvalidParam.AddDesc("empty branch name") return } - remoteName := c.Param("remoteName") + remoteName := c.Query("remoteName") if remoteName == "" { ctx.Err = e.ErrInvalidParam.AddDesc("empty remote name") return @@ -119,12 +113,17 @@ func GetGitRepoInfo(c *gin.Context) { dir := c.Query("dir") - ctx.Resp, ctx.Err = service.GetGitRepoInfo(codehostID, c.Query("repoOwner"), repoName, branchName, remoteName, dir, ctx.Logger) + namespace := c.Query("repoNamespace") + if namespace == "" { + namespace = c.Query("repoOwner") + } + ctx.Resp, ctx.Err = service.GetGitRepoInfo(codehostID, c.Query("repoOwner"), namespace, repoName, branchName, remoteName, dir, ctx.Logger) } type repoInfo struct { CodeHostID int `json:"codehost_id" form:"codehost_id"` Owner string `json:"owner" form:"owner"` + Namespace string `json:"namespace" form:"namespace"` Repo string `json:"repo" form:"repo"` Path string `json:"path" form:"path"` Branch string `json:"branch" form:"branch"` @@ -146,7 +145,12 @@ func GetRepoTree(c *gin.Context) { return } - ctx.Resp, ctx.Err = service.GetRepoTree(info.CodeHostID, info.Owner, info.Repo, info.Path, info.Branch, ctx.Logger) + owner := info.Owner + if len(info.Namespace) > 0 { + owner = info.Namespace + } + + ctx.Resp, ctx.Err = service.GetRepoTree(info.CodeHostID, owner, info.Repo, info.Path, info.Branch, ctx.Logger) } func GetCodehubRepoInfo(c *gin.Context) { @@ -165,13 +169,13 @@ func GetCodehubRepoInfo(c *gin.Context) { return } - repoUUID := c.Param("repoUUID") + repoUUID := c.Query("repoUUID") if repoUUID == "" { ctx.Err = e.ErrInvalidParam.AddDesc("empty repo uuid") return } - branchName := c.Param("branchName") + branchName := c.Query("branchName") if branchName == "" { ctx.Err = e.ErrInvalidParam.AddDesc("empty branch name") return @@ -181,3 +185,31 @@ func GetCodehubRepoInfo(c *gin.Context) { ctx.Resp, ctx.Err = service.GetCodehubRepoInfo(codehostID, repoUUID, branchName, path, ctx.Logger) } + +func GetContents(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + codehostIDStr := c.Param("codehostId") + codehostID, err := strconv.Atoi(codehostIDStr) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("cannot convert codehost id to int") + return + } + + repoName := c.Query("repoName") + if repoName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("repoName cannot be empty") + return + } + branchName := c.Query("branchName") + path := c.Query("path") + isDir, err := strconv.ParseBool(c.Query("isDir")) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalidParam isDir") + return + } + repoOwner := c.Query("repoOwner") + + ctx.Resp, ctx.Err = service.GetContents(codehostID, repoOwner, repoName, path, branchName, isDir, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/code/service/branch.go b/pkg/microservice/aslan/core/code/service/branch.go index b11b334405fa1020db0eb342b9380aff5c032367..4079c58337236d8a44bc923401f76cfe9a24284f 100644 --- a/pkg/microservice/aslan/core/code/service/branch.go +++ b/pkg/microservice/aslan/core/code/service/branch.go @@ -17,69 +17,32 @@ limitations under the License. package service import ( - "context" - "go.uber.org/zap" - "github.com/koderover/zadig/pkg/microservice/aslan/config" - git "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/github" - "github.com/koderover/zadig/pkg/shared/codehost" - "github.com/koderover/zadig/pkg/tool/codehub" - e "github.com/koderover/zadig/pkg/tool/errors" - "github.com/koderover/zadig/pkg/tool/gerrit" - "github.com/koderover/zadig/pkg/tool/git/gitlab" - "github.com/koderover/zadig/pkg/tool/ilyshin" + "github.com/koderover/zadig/pkg/microservice/aslan/core/code/client" + "github.com/koderover/zadig/pkg/microservice/aslan/core/code/client/open" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" ) -func CodeHostListBranches(codeHostID int, projectName, namespace string, log *zap.SugaredLogger) ([]*Branch, error) { - opt := &codehost.Option{ - CodeHostID: codeHostID, +func CodeHostListBranches(codeHostID int, projectName, namespace, key string, page, perPage int, log *zap.SugaredLogger) ([]*client.Branch, error) { + ch, err := systemconfig.New().GetCodeHost(codeHostID) + if err != nil { + log.Errorf("get code host info err:%s", err) + return nil, err + } + if ch.Type == setting.SourceFromOther { + return []*client.Branch{}, nil } - ch, err := codehost.GetCodeHostInfo(opt) + cli, err := open.OpenClient(ch, log) if err != nil { - return nil, e.ErrCodehostListBranches.AddDesc("git client is nil") + log.Errorf("open client err:%s", err) + return nil, err } - - if ch.Type == codeHostGitlab { - client, err := gitlab.NewClient(ch.Address, ch.AccessToken) - if err != nil { - log.Errorf("get gitlab client failed, err:%v", err) - return nil, e.ErrCodehostListBranches.AddDesc(err.Error()) - } - - brList, err := client.ListBranches(namespace, projectName, nil) - if err != nil { - return nil, err - } - return ToBranches(brList), nil - } else if ch.Type == CodeHostIlyshin { - client := ilyshin.NewClient(ch.Address, ch.AccessToken) - brList, err := client.ListBranches(namespace, projectName, log) - if err != nil { - return nil, err - } - return ToBranches(brList), nil - } else if ch.Type == gerrit.CodehostTypeGerrit { - cli := gerrit.NewClient(ch.Address, ch.AccessToken) - branches, err := cli.ListBranches(projectName) - if err != nil { - return nil, err - } - return ToBranches(branches), nil - } else if ch.Type == CodeHostCodeHub { - codeHubClient := codehub.NewCodeHubClient(ch.AccessKey, ch.SecretKey, ch.Region) - branchList, err := codeHubClient.BranchList(projectName) - if err != nil { - return nil, err - } - return ToBranches(branchList), nil - } else { - // github - gh := git.NewClient(ch.AccessToken, config.ProxyHTTPSAddr()) - branches, err := gh.ListBranches(context.TODO(), namespace, projectName, nil) - if err != nil { - return nil, err - } - return ToBranches(branches), nil + br, err := cli.ListBranches(client.ListOpt{Namespace: namespace, ProjectName: projectName, Key: key, Page: page, PerPage: perPage}) + if err != nil { + log.Errorf("list branch err:%s", err) + return nil, err } + return br, nil } diff --git a/pkg/microservice/aslan/core/code/service/merge_request.go b/pkg/microservice/aslan/core/code/service/merge_request.go index 527bb06bcfce27e81d2089d64ae381dd3f7853a8..e65f3ec911a5d58491732e4ec3818cc0c7f50d53 100644 --- a/pkg/microservice/aslan/core/code/service/merge_request.go +++ b/pkg/microservice/aslan/core/code/service/merge_request.go @@ -17,62 +17,39 @@ limitations under the License. package service import ( - "context" - "go.uber.org/zap" - "github.com/koderover/zadig/pkg/microservice/aslan/config" - git "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/github" - "github.com/koderover/zadig/pkg/shared/codehost" - e "github.com/koderover/zadig/pkg/tool/errors" - "github.com/koderover/zadig/pkg/tool/gerrit" - "github.com/koderover/zadig/pkg/tool/git/gitlab" - "github.com/koderover/zadig/pkg/tool/ilyshin" + "github.com/koderover/zadig/pkg/microservice/aslan/core/code/client" + "github.com/koderover/zadig/pkg/microservice/aslan/core/code/client/open" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" ) -func CodeHostListPRs(codeHostID int, projectName, namespace, targetBr string, log *zap.SugaredLogger) ([]*PullRequest, error) { - opt := &codehost.Option{ - CodeHostID: codeHostID, +func CodeHostListPRs(codeHostID int, projectName, namespace, targetBr string, key string, page, perPage int, log *zap.SugaredLogger) ([]*client.PullRequest, error) { + ch, err := systemconfig.New().GetCodeHost(codeHostID) + if err != nil { + log.Errorf("get code host info err:%s", err) + return nil, err } - ch, err := codehost.GetCodeHostInfo(opt) + if ch.Type == setting.SourceFromOther { + return []*client.PullRequest{}, nil + } + cli, err := open.OpenClient(ch, log) if err != nil { - return nil, e.ErrCodehostListPrs.AddDesc("git client is nil") + log.Errorf("open client err:%s", err) + return nil, err } - - if ch.Type == codeHostGitlab { - client, err := gitlab.NewClient(ch.Address, ch.AccessToken) - if err != nil { - log.Error(err) - return nil, e.ErrCodehostListPrs.AddDesc(err.Error()) - } - - prs, err := client.ListOpenedProjectMergeRequests(namespace, projectName, targetBr, nil) - if err != nil { - log.Error(err) - return nil, e.ErrCodehostListPrs.AddDesc(err.Error()) - } - - return ToPullRequests(prs), nil - - } else if ch.Type == CodeHostIlyshin { - client := ilyshin.NewClient(ch.Address, ch.AccessToken) - prs, err := client.ListOpenedProjectMergeRequests(namespace, projectName, targetBr, log) - if err != nil { - log.Error(err) - return nil, e.ErrCodehostListPrs.AddDesc(err.Error()) - } - return ToPullRequests(prs), nil - } else if ch.Type == gerrit.CodehostTypeGerrit { - return nil, nil - } else if ch.Type == CodeHostCodeHub { - return nil, nil - } else { - // github - gh := git.NewClient(ch.AccessToken, config.ProxyHTTPSAddr()) - pullRequests, err := gh.ListPullRequests(context.TODO(), namespace, projectName, nil) - if err != nil { - return nil, err - } - return ToPullRequests(pullRequests), nil + prs, err := cli.ListPrs(client.ListOpt{ + Namespace: namespace, + ProjectName: projectName, + Key: key, + Page: page, + PerPage: perPage, + TargeBr: targetBr, + }) + if err != nil { + log.Errorf("list prs err:%s", err) + return nil, err } + return prs, nil } diff --git a/pkg/microservice/aslan/core/code/service/namespace.go b/pkg/microservice/aslan/core/code/service/namespace.go index 68eb68ec27eb5f938c641649cfc1fd09f97456b0..4e6107281a7b65b2ba291da1aa84a201bc331dd5 100644 --- a/pkg/microservice/aslan/core/code/service/namespace.go +++ b/pkg/microservice/aslan/core/code/service/namespace.go @@ -17,18 +17,12 @@ limitations under the License. package service import ( - "context" - "go.uber.org/zap" - "github.com/koderover/zadig/pkg/microservice/aslan/config" - git "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/github" - "github.com/koderover/zadig/pkg/shared/codehost" - "github.com/koderover/zadig/pkg/tool/codehub" - e "github.com/koderover/zadig/pkg/tool/errors" - "github.com/koderover/zadig/pkg/tool/gerrit" - "github.com/koderover/zadig/pkg/tool/git/gitlab" - "github.com/koderover/zadig/pkg/tool/ilyshin" + "github.com/koderover/zadig/pkg/microservice/aslan/core/code/client" + "github.com/koderover/zadig/pkg/microservice/aslan/core/code/client/open" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" ) const ( @@ -36,70 +30,29 @@ const ( OrgKind = "org" GroupKind = "group" UserKind = "user" - page = 1 - perPage = 100 CodeHostCodeHub = "codehub" - CodeHostIlyshin = "ilyshin" ) -func CodeHostListNamespaces(codeHostID int, keyword string, log *zap.SugaredLogger) ([]*Namespace, error) { - opt := &codehost.Option{ - CodeHostID: codeHostID, - } - ch, err := codehost.GetCodeHostInfo(opt) +func CodeHostListNamespaces(codeHostID int, keyword string, log *zap.SugaredLogger) ([]*client.Namespace, error) { + ch, err := systemconfig.New().GetCodeHost(codeHostID) if err != nil { - return nil, e.ErrCodehostListNamespaces.AddDesc("git client is nil") + log.Errorf("get code host info err:%s", err) + return nil, err } - if ch.Type == codeHostGitlab { - client, err := gitlab.NewClient(ch.Address, ch.AccessToken) - if err != nil { - log.Error(err) - return nil, e.ErrCodehostListNamespaces.AddDesc(err.Error()) - } - - nsList, err := client.ListNamespaces(keyword, nil) - if err != nil { - return nil, err - } - return ToNamespaces(nsList), nil - } else if ch.Type == CodeHostIlyshin { - client := ilyshin.NewClient(ch.Address, ch.AccessToken) - nsList, err := client.ListNamespaces(keyword, log) - if err != nil { - return nil, err - } - return ToNamespaces(nsList), nil - } else if ch.Type == gerrit.CodehostTypeGerrit { - return []*Namespace{{ - Name: gerrit.DefaultNamespace, - Path: gerrit.DefaultNamespace, - Kind: OrgKind, - }}, nil - } else if ch.Type == CodeHostCodeHub { - codeHubClient := codehub.NewCodeHubClient(ch.AccessKey, ch.SecretKey, ch.Region) - codeHubNamespace, err := codeHubClient.NamespaceList() - if err != nil { - return nil, err - } - return ToNamespaces(codeHubNamespace), nil - } else { - // github - gh := git.NewClient(ch.AccessToken, config.ProxyHTTPSAddr()) - namespaces := make([]*Namespace, 0) - - user, err := gh.GetAuthenticatedUser(context.TODO()) - if err != nil { - return nil, err - } - namespaces = append(namespaces, ToNamespaces(user)...) - - organizations, err := gh.ListOrganizationsForAuthenticatedUser(context.TODO(), nil) - if err != nil { - return nil, err - } - namespaces = append(namespaces, ToNamespaces(organizations)...) + if ch.Type == setting.SourceFromOther { + return []*client.Namespace{}, nil + } - return namespaces, nil + cli, err := open.OpenClient(ch, log) + if err != nil { + log.Errorf("open client err:%s", err) + return nil, err + } + ns, err := cli.ListNamespaces(keyword) + if err != nil { + log.Errorf("list namespace err:%s", err) + return nil, err } + return ns, nil } diff --git a/pkg/microservice/aslan/core/code/service/project.go b/pkg/microservice/aslan/core/code/service/project.go index 1c724afbc2257ffe5f1bd47fa47c582fb7a47e4f..3596c46b32cdb161a6ca70cd0e20395ab5f4cd5a 100644 --- a/pkg/microservice/aslan/core/code/service/project.go +++ b/pkg/microservice/aslan/core/code/service/project.go @@ -17,92 +17,38 @@ limitations under the License. package service import ( - "context" - "go.uber.org/zap" - "github.com/koderover/zadig/pkg/microservice/aslan/config" - git "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/github" - "github.com/koderover/zadig/pkg/shared/codehost" - "github.com/koderover/zadig/pkg/tool/codehub" - e "github.com/koderover/zadig/pkg/tool/errors" - "github.com/koderover/zadig/pkg/tool/gerrit" - "github.com/koderover/zadig/pkg/tool/git/gitlab" - "github.com/koderover/zadig/pkg/tool/ilyshin" + "github.com/koderover/zadig/pkg/microservice/aslan/core/code/client" + "github.com/koderover/zadig/pkg/microservice/aslan/core/code/client/open" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" ) -func CodeHostListProjects(codeHostID int, namespace, namespaceType, keyword string, log *zap.SugaredLogger) ([]*Project, error) { - opt := &codehost.Option{ - CodeHostID: codeHostID, +func CodeHostListProjects(codeHostID int, namespace, namespaceType string, page, perPage int, keyword string, log *zap.SugaredLogger) ([]*client.Project, error) { + ch, err := systemconfig.New().GetCodeHost(codeHostID) + if err != nil { + log.Errorf("get code host info err:%s", err) + return nil, err } - ch, err := codehost.GetCodeHostInfo(opt) + if ch.Type == setting.SourceFromOther { + return []*client.Project{}, nil + } + cli, err := open.OpenClient(ch, log) if err != nil { - log.Error(err) - return nil, e.ErrCodehostListProjects.AddDesc("git client is nil") + log.Errorf("open client err:%s", err) + return nil, err } - if ch.Type == codeHostGitlab { - client, err := gitlab.NewClient(ch.Address, ch.AccessToken) - if err != nil { - log.Error(err) - return nil, e.ErrCodehostListProjects.AddDesc(err.Error()) - } - - if namespaceType == GroupKind { - projects, err := client.ListGroupProjects(namespace, keyword, nil) - if err != nil { - log.Error(err) - return nil, e.ErrCodehostListProjects.AddDesc(err.Error()) - } - return ToProjects(projects), nil - } - - // user - projects, err := client.ListUserProjects(namespace, keyword, nil) - if err != nil { - log.Error(err) - return nil, e.ErrCodehostListProjects.AddDesc(err.Error()) - } - return ToProjects(projects), nil - - } else if ch.Type == CodeHostIlyshin { - cli := ilyshin.NewClient(ch.Address, ch.AccessToken) - projects, err := cli.ListGroupProjects(namespace, keyword, log) - if err != nil { - log.Error(err) - return nil, e.ErrCodehostListProjects.AddDesc(err.Error()) - } - return ToProjects(projects), nil - } else if ch.Type == gerrit.CodehostTypeGerrit { - cli := gerrit.NewClient(ch.Address, ch.AccessToken) - projects, err := cli.ListProjectsByKey(keyword) - if err != nil { - log.Error(err) - return nil, e.ErrCodehostListProjects.AddDesc(err.Error()) - } - return ToProjects(projects), nil - } else if ch.Type == CodeHostCodeHub { - codeHubClient := codehub.NewCodeHubClient(ch.AccessKey, ch.SecretKey, ch.Region) - projects, err := codeHubClient.RepoList(namespace, keyword, 100) - if err != nil { - log.Error(err) - return nil, e.ErrCodehostListProjects.AddDesc(err.Error()) - } - return ToProjects(projects), nil - } else { - // github - gh := git.NewClient(ch.AccessToken, config.ProxyHTTPSAddr()) - repos, err := gh.ListRepositoriesForAuthenticatedUser(context.TODO(), nil) - if err != nil { - return nil, err - } - var projects []*Project - for _, p := range ToProjects(repos) { - if p.Namespace != namespace { - continue - } - projects = append(projects, p) - } - - return projects, nil + projects, err := cli.ListProjects(client.ListOpt{ + Namespace: namespace, + NamespaceType: namespaceType, + Key: keyword, + Page: page, + PerPage: perPage, + }) + if err != nil { + log.Errorf("list projects err:%s", err) + return nil, err } + return projects, nil } diff --git a/pkg/microservice/aslan/core/code/service/repo.go b/pkg/microservice/aslan/core/code/service/repo.go index 5fe2392f126bdb437b5d165266c185f65412ddab..8f2cd5fb581ec31811621e935e972bd6266bad91 100644 --- a/pkg/microservice/aslan/core/code/service/repo.go +++ b/pkg/microservice/aslan/core/code/service/repo.go @@ -17,11 +17,17 @@ limitations under the License. package service import ( + "regexp" "strings" "sync" "github.com/hashicorp/go-multierror" "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/code/client" + "github.com/koderover/zadig/pkg/microservice/aslan/core/code/client/open" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" ) type RepoInfoList struct { @@ -29,43 +35,65 @@ type RepoInfoList struct { } type GitRepoInfo struct { - Owner string `json:"repo_owner"` - Repo string `json:"repo"` - CodehostID int `json:"codehost_id"` - Source string `json:"source"` - DefaultBranch string `json:"default_branch"` - ErrorMsg string `json:"error_msg"` // repo信息是否拉取成功 - Branches []*Branch `json:"branches"` - Tags []*Tag `json:"tags"` - PRs []*PullRequest `json:"prs"` - ProjectUUID string `json:"project_uuid,omitempty"` - RepoUUID string `json:"repo_uuid,omitempty"` - RepoID string `json:"repo_id,omitempty"` + Owner string `json:"repo_owner"` + Namespace string `json:"repo_namespace"` + Repo string `json:"repo"` + CodehostID int `json:"codehost_id"` + Source string `json:"source"` + DefaultBranch string `json:"default_branch"` + ErrorMsg string `json:"error_msg"` // get repo message fail message + Branches []*client.Branch `json:"branches"` + Tags []*client.Tag `json:"tags"` + PRs []*client.PullRequest `json:"prs"` + ProjectUUID string `json:"project_uuid,omitempty"` + RepoUUID string `json:"repo_uuid,omitempty"` + RepoID string `json:"repo_id,omitempty"` + Key string `json:"key"` + // FilterRegexp is the regular expression filter for the branches and tags + FilterRegexp string `json:"filter_regexp,omitempty"` +} + +func (repo *GitRepoInfo) GetNamespace() string { + if repo.Namespace == "" { + return repo.Owner + } + return repo.Namespace } // ListRepoInfos ... -func ListRepoInfos(infos []*GitRepoInfo, param string, log *zap.SugaredLogger) ([]*GitRepoInfo, error) { - var err error +func ListRepoInfos(infos []*GitRepoInfo, log *zap.SugaredLogger) ([]*GitRepoInfo, error) { var wg sync.WaitGroup var errList *multierror.Error for _, info := range infos { - //pb 代表pr and branch - if param == "" || param == "bp" { - wg.Add(1) - go func(info *GitRepoInfo) { - defer func() { - wg.Done() - }() - info.PRs, err = CodeHostListPRs(info.CodehostID, info.Repo, strings.Replace(info.Owner, "%2F", "/", -1), "", log) - if err != nil { - errList = multierror.Append(errList, err) - info.ErrorMsg = err.Error() - info.PRs = []*PullRequest{} - return - } - }(info) + ch, err := systemconfig.New().GetCodeHost(info.CodehostID) + if err != nil { + log.Errorf("get code host info err:%s", err) + return nil, err + } + if ch.Type == setting.SourceFromOther { + continue } + codehostClient, err := open.OpenClient(ch, log) + if err != nil { + return nil, err + } + wg.Add(1) + go func(info *GitRepoInfo) { + defer func() { + wg.Done() + }() + info.PRs, err = codehostClient.ListPrs(client.ListOpt{ + Namespace: strings.Replace(info.GetNamespace(), "%2F", "/", -1), + ProjectName: info.Repo, + }) + if err != nil { + errList = multierror.Append(errList, err) + info.ErrorMsg = err.Error() + info.PRs = []*client.PullRequest{} + return + } + }(info) wg.Add(1) go func(info *GitRepoInfo) { @@ -76,41 +104,95 @@ func ListRepoInfos(infos []*GitRepoInfo, param string, log *zap.SugaredLogger) ( if info.Source == CodeHostCodeHub { projectName = info.RepoUUID } - info.Branches, err = CodeHostListBranches(info.CodehostID, projectName, strings.Replace(info.Owner, "%2F", "/", -1), log) + info.Branches, err = codehostClient.ListBranches(client.ListOpt{ + Namespace: strings.Replace(info.GetNamespace(), "%2F", "/", -1), + ProjectName: projectName, + Key: info.Key, + }) if err != nil { errList = multierror.Append(errList, err) info.ErrorMsg = err.Error() - info.Branches = []*Branch{} + info.Branches = []*client.Branch{} return } }(info) - //bt 代表branch and tag - if param == "" || param == "bt" { - wg.Add(1) - go func(info *GitRepoInfo) { - defer func() { - wg.Done() - }() - projectName := info.Repo - if info.Source == CodeHostCodeHub { - projectName = info.RepoID + wg.Add(1) + go func(info *GitRepoInfo) { + defer func() { + wg.Done() + }() + projectName := info.Repo + if info.Source == CodeHostCodeHub { + projectName = info.RepoID + } + + info.Tags, err = codehostClient.ListTags(client.ListOpt{ + Namespace: strings.Replace(info.GetNamespace(), "%2F", "/", -1), + ProjectName: projectName, + Key: info.Key, + }) + if err != nil { + errList = multierror.Append(errList, err) + info.ErrorMsg = err.Error() + info.Tags = []*client.Tag{} + return + } + }(info) + } + + wg.Wait() + for _, info := range infos { + if info.FilterRegexp != "" { + newBranchList := make([]*client.Branch, 0) + for _, branch := range info.Branches { + match, err := regexp.MatchString(info.FilterRegexp, branch.Name) + if err != nil { + log.Errorf("failed to match regular expression: %s on branch name :%s, error: %s", info.FilterRegexp, branch.Name, err) + return nil, err + } + if match { + newBranchList = append(newBranchList, branch) } - info.Tags, err = CodeHostListTags(info.CodehostID, projectName, strings.Replace(info.Owner, "%2F", "/", -1), log) + } + newTagList := make([]*client.Tag, 0) + for _, tag := range info.Tags { + match, err := regexp.MatchString(info.FilterRegexp, tag.Name) if err != nil { - errList = multierror.Append(errList, err) - info.ErrorMsg = err.Error() - info.Tags = []*Tag{} - return + log.Errorf("failed to match regular expression: %s on tag name :%s, error: %s", info.FilterRegexp, tag.Name, err) + return nil, err + } + if match { + newTagList = append(newTagList, tag) } - }(info) + } + info.Branches = newBranchList + info.Tags = newTagList + + match, err := regexp.MatchString(info.FilterRegexp, info.DefaultBranch) + if err != nil { + log.Errorf("failed to compile regular expression: %s, the error is: %s", info.FilterRegexp, err) + return nil, err + } + if !match { + info.DefaultBranch = "" + } } } - - wg.Wait() if err := errList.ErrorOrNil(); err != nil { log.Errorf("list repo info error: %v", err) + return nil, err } return infos, nil } + +func MatchBranchesList(regular string, branches []string) []string { + matchBranches := make([]string, 0) + for _, branch := range branches { + if matched, _ := regexp.MatchString(regular, branch); matched { + matchBranches = append(matchBranches, branch) + } + } + return matchBranches +} diff --git a/pkg/microservice/aslan/core/code/service/tag.go b/pkg/microservice/aslan/core/code/service/tag.go index 8c3edcaafecb38a175b54a826b52d52d5b631914..79039182a5f7150e015fdda7c32433e3c7f954eb 100644 --- a/pkg/microservice/aslan/core/code/service/tag.go +++ b/pkg/microservice/aslan/core/code/service/tag.go @@ -17,72 +17,38 @@ limitations under the License. package service import ( - "context" - "go.uber.org/zap" - "github.com/koderover/zadig/pkg/microservice/aslan/config" - git "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/github" - "github.com/koderover/zadig/pkg/shared/codehost" - "github.com/koderover/zadig/pkg/tool/codehub" - e "github.com/koderover/zadig/pkg/tool/errors" - "github.com/koderover/zadig/pkg/tool/gerrit" - "github.com/koderover/zadig/pkg/tool/git/gitlab" - "github.com/koderover/zadig/pkg/tool/ilyshin" + "github.com/koderover/zadig/pkg/microservice/aslan/core/code/client" + "github.com/koderover/zadig/pkg/microservice/aslan/core/code/client/open" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" ) -func CodeHostListTags(codeHostID int, projectName string, namespace string, log *zap.SugaredLogger) ([]*Tag, error) { - opt := &codehost.Option{ - CodeHostID: codeHostID, +func CodeHostListTags(codeHostID int, projectName string, namespace string, key string, page int, perPage int, log *zap.SugaredLogger) ([]*client.Tag, error) { + ch, err := systemconfig.New().GetCodeHost(codeHostID) + if err != nil { + log.Errorf("get code host info err:%s", err) + return nil, err } - ch, err := codehost.GetCodeHostInfo(opt) + if ch.Type == setting.SourceFromOther { + return []*client.Tag{}, nil + } + cli, err := open.OpenClient(ch, log) if err != nil { - log.Error(err) - return nil, e.ErrCodehostListTags.AddDesc("git client is nil") + log.Errorf("open client err:%s", err) + return nil, err } - - if ch.Type == codeHostGitlab { - client, err := gitlab.NewClient(ch.Address, ch.AccessToken) - if err != nil { - log.Error(err) - return nil, e.ErrCodehostListTags.AddDesc(err.Error()) - } - - tags, err := client.ListTags(namespace, projectName, nil) - if err != nil { - return nil, err - } - return ToTags(tags), nil - } else if ch.Type == CodeHostIlyshin { - client := ilyshin.NewClient(ch.Address, ch.AccessToken) - tags, err := client.ListTags(namespace, projectName, log) - if err != nil { - return nil, err - } - return ToTags(tags), nil - } else if ch.Type == gerrit.CodehostTypeGerrit { - client := gerrit.NewClient(ch.Address, ch.AccessToken) - tags, err := client.ListTags(projectName) - if err != nil { - return nil, err - } - - return ToTags(tags), nil - } else if ch.Type == CodeHostCodeHub { - codeHubClient := codehub.NewCodeHubClient(ch.AccessKey, ch.SecretKey, ch.Region) - tags, err := codeHubClient.TagList(projectName) - if err != nil { - return nil, err - } - return ToTags(tags), nil - } else { - // github - gh := git.NewClient(ch.AccessToken, config.ProxyHTTPSAddr()) - tags, err := gh.ListTags(context.TODO(), namespace, projectName, nil) - if err != nil { - return nil, err - } - - return ToTags(tags), nil + tags, err := cli.ListTags(client.ListOpt{ + Namespace: namespace, + ProjectName: projectName, + Key: key, + Page: page, + PerPage: page, + }) + if err != nil { + log.Errorf("list tags err:%s", err) + return nil, err } + return tags, nil } diff --git a/pkg/microservice/aslan/core/code/service/types.go b/pkg/microservice/aslan/core/code/service/types.go deleted file mode 100644 index 722c415a71175ee43b4ed4b4d457fe86c0ea345c..0000000000000000000000000000000000000000 --- a/pkg/microservice/aslan/core/code/service/types.go +++ /dev/null @@ -1,332 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package service - -import ( - "strings" - - "github.com/andygrunwald/go-gerrit" - "github.com/google/go-github/v35/github" - "github.com/koderover/zadig/pkg/tool/ilyshin" - "github.com/xanzy/go-gitlab" - - "github.com/koderover/zadig/pkg/tool/codehub" - gerrittool "github.com/koderover/zadig/pkg/tool/gerrit" -) - -type Branch struct { - Name string `json:"name"` - Protected bool `json:"protected"` - Merged bool `json:"merged"` -} - -type PullRequest struct { - ID int `json:"id"` - TargetBranch string `json:"targetBranch"` - SourceBranch string `json:"sourceBranch"` - ProjectID int `json:"projectId"` - Title string `json:"title"` - State string `json:"state"` - CreatedAt int64 `json:"createdAt"` - UpdatedAt int64 `json:"updatedAt"` - AuthorUsername string `json:"authorUsername"` - Number int `json:"number"` - User string `json:"user"` - Base string `json:"base,omitempty"` -} - -type Namespace struct { - Name string `json:"name"` - Path string `json:"path"` - Kind string `json:"kind"` - ProjectUUID string `json:"project_uuid,omitempty"` -} - -type Project struct { - ID int `json:"id"` - Name string `json:"name"` - Description string `json:"description"` - DefaultBranch string `json:"defaultBranch"` - Namespace string `json:"namespace"` - RepoUUID string `json:"repo_uuid,omitempty"` - RepoID string `json:"repo_id,omitempty"` -} - -type Tag struct { - Name string `json:"name"` - ZipballURL string `json:"zipball_url"` - TarballURL string `json:"tarball_url"` - Message string `json:"message"` -} - -func ToBranches(obj interface{}) []*Branch { - var res []*Branch - - switch os := obj.(type) { - case []*github.Branch: - for _, o := range os { - res = append(res, &Branch{ - Name: o.GetName(), - Protected: o.GetProtected(), - }) - } - case []*gitlab.Branch: - for _, o := range os { - res = append(res, &Branch{ - Name: o.Name, - Protected: o.Protected, - Merged: o.Merged, - }) - } - case []string: - for _, o := range os { - res = append(res, &Branch{ - Name: o, - }) - } - case []*codehub.Branch: - for _, o := range os { - res = append(res, &Branch{ - Name: o.Name, - Protected: o.Protected, - Merged: o.Merged, - }) - } - case []*ilyshin.Branch: - for _, o := range os { - res = append(res, &Branch{ - Name: o.Name, - Protected: o.Protected, - Merged: o.Merged, - }) - } - } - - return res -} - -func ToPullRequests(obj interface{}) []*PullRequest { - var res []*PullRequest - - switch os := obj.(type) { - case []*github.PullRequest: - for _, o := range os { - res = append(res, &PullRequest{ - ID: o.GetNumber(), - CreatedAt: o.GetCreatedAt().Unix(), - UpdatedAt: o.GetUpdatedAt().Unix(), - State: o.GetState(), - User: o.GetUser().GetLogin(), - Number: o.GetNumber(), - AuthorUsername: o.GetUser().GetLogin(), - Title: o.GetTitle(), - SourceBranch: o.GetHead().GetRef(), - TargetBranch: o.GetBase().GetRef(), - }) - } - case []*gitlab.MergeRequest: - for _, o := range os { - res = append(res, &PullRequest{ - ID: o.IID, - TargetBranch: o.TargetBranch, - SourceBranch: o.SourceBranch, - ProjectID: o.ProjectID, - Title: o.Title, - State: o.State, - CreatedAt: o.CreatedAt.Unix(), - UpdatedAt: o.UpdatedAt.Unix(), - AuthorUsername: o.Author.Username, - }) - } - case []*ilyshin.MergeRequest: - for _, o := range os { - res = append(res, &PullRequest{ - ID: o.IID, - TargetBranch: o.TargetBranch, - SourceBranch: o.SourceBranch, - ProjectID: o.ProjectID, - Title: o.Title, - State: o.State, - CreatedAt: o.CreatedAt.Unix(), - UpdatedAt: o.UpdatedAt.Unix(), - AuthorUsername: o.Author.Username, - }) - } - } - - return res -} - -func ToNamespaces(obj interface{}) []*Namespace { - var res []*Namespace - - switch os := obj.(type) { - case *github.User: - res = append(res, &Namespace{ - Name: os.GetLogin(), - Path: os.GetLogin(), - Kind: UserKind, - }) - case []*github.User: - for _, o := range os { - res = append(res, &Namespace{ - Name: o.GetLogin(), - Path: o.GetLogin(), - Kind: UserKind, - }) - } - case []*github.Organization: - for _, o := range os { - res = append(res, &Namespace{ - Name: o.GetLogin(), - Path: o.GetLogin(), - Kind: OrgKind, - }) - } - case []*gitlab.Namespace: - for _, o := range os { - res = append(res, &Namespace{ - Name: o.Path, - Path: o.FullPath, - Kind: o.Kind, - }) - } - case []*codehub.Namespace: - for _, o := range os { - res = append(res, &Namespace{ - Name: o.Name, - Path: o.Path, - Kind: o.Kind, - ProjectUUID: o.ProjectUUID, - }) - } - case []*ilyshin.Project: - for _, o := range os { - res = append(res, &Namespace{ - Name: o.Namespace.Name, - Path: o.Namespace.Path, - Kind: GroupKind, - }) - } - } - - return res -} - -func ToProjects(obj interface{}) []*Project { - var res []*Project - - switch os := obj.(type) { - case []*github.Repository: - for _, o := range os { - res = append(res, &Project{ - ID: int(o.GetID()), - Name: o.GetName(), - DefaultBranch: o.GetDefaultBranch(), - Namespace: o.GetOwner().GetLogin(), - }) - } - case []*gitlab.Project: - for _, o := range os { - res = append(res, &Project{ - ID: o.ID, - Name: o.Path, - Namespace: o.Namespace.FullPath, - Description: o.Description, - DefaultBranch: o.DefaultBranch, - }) - } - case []*gerrit.ProjectInfo: - for ind, o := range os { - res = append(res, &Project{ - ID: ind, // fake id - Name: gerrittool.Unescape(o.ID), // id could have %2F - Description: o.Description, - DefaultBranch: "master", - Namespace: gerrittool.DefaultNamespace, - }) - } - case []*codehub.Project: - for _, project := range os { - res = append(res, &Project{ - Name: project.Name, - Description: project.Description, - DefaultBranch: project.DefaultBranch, - Namespace: project.Namespace, - RepoUUID: project.RepoUUID, - RepoID: project.RepoID, - }) - } - case []*ilyshin.Project: - for _, o := range os { - namespaces := strings.Split(o.PathWithNamespace, "/") - res = append(res, &Project{ - ID: o.ID, - Name: o.Path, - Namespace: namespaces[0], - Description: o.Description, - DefaultBranch: "master", - }) - } - } - - return res -} - -func ToTags(obj interface{}) []*Tag { - var res []*Tag - - switch os := obj.(type) { - case []*github.RepositoryTag: - for _, o := range os { - res = append(res, &Tag{ - Name: o.GetName(), - ZipballURL: o.GetZipballURL(), - TarballURL: o.GetTarballURL(), - }) - } - case []*gitlab.Tag: - for _, o := range os { - res = append(res, &Tag{ - Name: o.Name, - Message: o.Message, - }) - } - case []*gerrit.TagInfo: - for _, o := range os { - res = append(res, &Tag{ - Name: o.Ref, - Message: o.Message, - }) - } - case []*codehub.Tag: - for _, o := range os { - res = append(res, &Tag{ - Name: o.Name, - }) - } - case []*ilyshin.Tag: - for _, o := range os { - res = append(res, &Tag{ - Name: o.Name, - Message: o.Message, - }) - } - } - - return res -} diff --git a/pkg/microservice/aslan/core/code/service/workspace.go b/pkg/microservice/aslan/core/code/service/workspace.go index 751e008ac1277423668907cde686b139441bd1e0..0f2a76200e7e66a3ff2a1f93f13bb76922d32b79 100644 --- a/pkg/microservice/aslan/core/code/service/workspace.go +++ b/pkg/microservice/aslan/core/code/service/workspace.go @@ -28,9 +28,10 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/command" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/fs" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/git" - "github.com/koderover/zadig/pkg/shared/codehost" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" "github.com/koderover/zadig/pkg/tool/codehub" e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/util" ) func GetPublicRepoTree(repoLink, path string, logger *zap.SugaredLogger) ([]*git.TreeNode, error) { @@ -95,10 +96,9 @@ func getWorkspaceBasePath(pipelineName string) (string, error) { return "", err } - //base := path.Join(s.Config.NFS.Path, pipe.Name) base := path.Join(config.S3StoragePath(), pipe.Name) - if _, err := os.Stat(base); os.IsNotExist(err) { + if exsit, err := util.PathExists(base); !exsit { return "", err } @@ -135,7 +135,7 @@ type FileInfo struct { IsDir bool `json:"is_dir"` } -func GetGitRepoInfo(codehostID int, repoOwner, repoName, branchName, remoteName, dir string, log *zap.SugaredLogger) ([]*FileInfo, error) { +func GetGitRepoInfo(codehostID int, repoOwner, repoNamespace, repoName, branchName, remoteName, dir string, log *zap.SugaredLogger) ([]*FileInfo, error) { fis := make([]*FileInfo, 0) if dir == "" { dir = "/" @@ -143,16 +143,16 @@ func GetGitRepoInfo(codehostID int, repoOwner, repoName, branchName, remoteName, base := path.Join(config.S3StoragePath(), repoName) if err := os.RemoveAll(base); err != nil { - log.Errorf("dir remove err:%v", err) + log.Warnf("dir remove err:%s", err) } - detail, err := codehost.GetCodehostDetail(codehostID) + detail, err := systemconfig.New().GetCodeHost(codehostID) if err != nil { - log.Errorf("GetGitRepoInfo GetCodehostDetail err:%v", err) + log.Errorf("GetGitRepoInfo GetCodehostDetail err:%s", err) return fis, e.ErrListRepoDir.AddDesc(err.Error()) } - err = command.RunGitCmds(detail, repoOwner, repoName, branchName, remoteName) + err = command.RunGitCmds(detail, repoOwner, repoNamespace, repoName, branchName, remoteName) if err != nil { - log.Errorf("GetGitRepoInfo runGitCmds err:%v", err) + log.Errorf("GetGitRepoInfo runGitCmds err:%s", err) return fis, e.ErrListRepoDir.AddDesc(err.Error()) } files, err := ioutil.ReadDir(path.Join(base, dir)) @@ -189,13 +189,13 @@ type CodehostFileInfo struct { func GetCodehubRepoInfo(codehostID int, repoUUID, branchName, path string, log *zap.SugaredLogger) ([]*CodehostFileInfo, error) { fileInfos := make([]*CodehostFileInfo, 0) - detail, err := codehost.GetCodehostDetail(codehostID) + detail, err := systemconfig.New().GetCodeHost(codehostID) if err != nil { log.Errorf("GetCodehubRepoInfo GetCodehostDetail err:%s", err) return fileInfos, e.ErrListWorkspace.AddDesc(err.Error()) } - codeHubClient := codehub.NewCodeHubClient(detail.AccessKey, detail.SecretKey, detail.Region) + codeHubClient := codehub.NewCodeHubClient(detail.AccessKey, detail.SecretKey, detail.Region, config.ProxyHTTPSAddr(), detail.EnableProxy) treeNodes, err := codeHubClient.FileTree(repoUUID, branchName, path) if err != nil { log.Errorf("Failed to list tree from codehub err:%s", err) @@ -211,3 +211,18 @@ func GetCodehubRepoInfo(codehostID int, repoUUID, branchName, path string, log * } return fileInfos, nil } + +func GetContents(codeHostID int, owner, repo, path, branch string, isDir bool, logger *zap.SugaredLogger) (string, error) { + getter, err := fs.GetTreeGetter(codeHostID) + if err != nil { + logger.Errorf("Failed to get tree getter, err: %s", err) + return "", e.ErrListWorkspace.AddDesc(err.Error()) + } + + yamlInfos, err := getter.GetYAMLContents(owner, repo, path, branch, isDir, false) + if err != nil { + return "", e.ErrListWorkspace.AddDesc(err.Error()) + } + + return util.CombineManifests(yamlInfos), nil +} diff --git a/pkg/microservice/aslan/core/collaboration/config/consts.go b/pkg/microservice/aslan/core/collaboration/config/consts.go new file mode 100644 index 0000000000000000000000000000000000000000..c51ed2e14962887101333f1463f4fdc0669e37a1 --- /dev/null +++ b/pkg/microservice/aslan/core/collaboration/config/consts.go @@ -0,0 +1,24 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +type CollaborationType string + +const ( + CollaborationShare CollaborationType = "share" + CollaborationNew CollaborationType = "new" +) diff --git a/pkg/microservice/aslan/core/collaboration/handler/collaboration_instance.go b/pkg/microservice/aslan/core/collaboration/handler/collaboration_instance.go new file mode 100644 index 0000000000000000000000000000000000000000..416f44240733f3757ee5fe2b4d27fd2afc433aea --- /dev/null +++ b/pkg/microservice/aslan/core/collaboration/handler/collaboration_instance.go @@ -0,0 +1,68 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "encoding/json" + + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/collaboration/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/log" +) + +func GetCollaborationNew(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("projectName can not be empty") + return + } + ctx.Resp, ctx.Err = service.GetCollaborationNew(projectName, ctx.UserID, ctx.IdentityType, ctx.Account, ctx.Logger) +} + +func SyncCollaborationInstance(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + projectName := c.Query("projectName") + args := new(service.SyncCollaborationInstanceArgs) + data, err := c.GetRawData() + if err != nil { + log.Errorf("SyncCollaborationInstance c.GetRawData() err: %s", err) + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("SyncCollaborationInstance json.Unmarshal err: %s", err) + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("projectName can not be empty") + return + } + ctx.Err = service.SyncCollaborationInstance(args, projectName, ctx.UserID, ctx.IdentityType, ctx.Account, ctx.RequestID, ctx.Logger) +} + +func CleanCIResources(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + ctx.Err = service.CleanCIResources(ctx.UserName, ctx.RequestID, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/collaboration/handler/collaboration_mode.go b/pkg/microservice/aslan/core/collaboration/handler/collaboration_mode.go new file mode 100644 index 0000000000000000000000000000000000000000..4658d6a148a5aa887d70f610d7e2804e9d259130 --- /dev/null +++ b/pkg/microservice/aslan/core/collaboration/handler/collaboration_mode.go @@ -0,0 +1,100 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "encoding/json" + + "github.com/gin-gonic/gin" + + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/collaboration/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/collaboration/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/collaboration" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/log" +) + +func GetCollaborationMode(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("projectName can not be empty") + return + } + ctx.Resp, ctx.Err = collaboration.GetCollaborationModes([]string{projectName}, ctx.Logger) +} + +func CreateCollaborationMode(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(commonmodels.CollaborationMode) + data, err := c.GetRawData() + if err != nil { + log.Errorf("CreateCollaborationMode c.GetRawData() err: %s", err) + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("CreateCollaborationMode json.Unmarshal err: %s", err) + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } + + internalhandler.InsertOperationLog(c, ctx.Account, args.ProjectName, "新增", "协作模式", args.Name, string(data), ctx.Logger) + + ctx.Err = service.CreateCollaborationMode(ctx.Account, args, ctx.Logger) +} + +func UpdateCollaborationMode(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(commonmodels.CollaborationMode) + data, err := c.GetRawData() + if err != nil { + log.Errorf("UpdateCollaborationMode c.GetRawData() err: %s", err) + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("UpdateCollaborationMode json.Unmarshal err: %s", err) + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } + + internalhandler.InsertOperationLog(c, ctx.Account, args.ProjectName, "更新", "协作模式", args.Name, string(data), ctx.Logger) + + ctx.Err = service.UpdateCollaborationMode(ctx.Account, args, ctx.Logger) +} + +func DeleteCollaborationMode(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("projectName can not be empty") + return + } + name := c.Param("name") + internalhandler.InsertOperationLog(c, ctx.Account, projectName, "删除", "协作模式", name, "", ctx.Logger) + + ctx.Err = service.DeleteCollaborationMode(ctx.Account, projectName, name, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/collaboration/handler/router.go b/pkg/microservice/aslan/core/collaboration/handler/router.go new file mode 100644 index 0000000000000000000000000000000000000000..5092b894cc355007e98d340b170a82b50f80e1c2 --- /dev/null +++ b/pkg/microservice/aslan/core/collaboration/handler/router.go @@ -0,0 +1,37 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" +) + +type Router struct{} + +func (*Router) Inject(router *gin.RouterGroup) { + + collaborations := router.Group("collaborations") + { + collaborations.GET("", GetCollaborationMode) + collaborations.PUT("/:name", UpdateCollaborationMode) + collaborations.POST("", CreateCollaborationMode) + collaborations.DELETE("/:name", DeleteCollaborationMode) + collaborations.GET("/cron/clean", CleanCIResources) + collaborations.GET("/new", GetCollaborationNew) + collaborations.POST("/sync", SyncCollaborationInstance) + } +} diff --git a/pkg/microservice/aslan/core/collaboration/repository/models/collaboration_instance.go b/pkg/microservice/aslan/core/collaboration/repository/models/collaboration_instance.go new file mode 100644 index 0000000000000000000000000000000000000000..5ecf7eae4cbad4bd92f33bf97bf3369fecf86c3d --- /dev/null +++ b/pkg/microservice/aslan/core/collaboration/repository/models/collaboration_instance.go @@ -0,0 +1,56 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/collaboration/config" +) + +type CollaborationInstance struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + ProjectName string `bson:"project_name" json:"project_name"` + CreateTime int64 `bson:"create_time" json:"create_time"` + UpdateTime int64 `bson:"update_time" json:"update_time"` + LastVisitTime int64 `bson:"last_visit_time" json:"last_visit_time"` + CollaborationName string `bson:"collaboration_name" json:"collaboration_name"` + RecycleDay int64 `bson:"recycle_day" json:"recycle_day"` + Revision int64 `bson:"revision" json:"revision"` + UserUID string `bson:"user_uid" json:"user_uid"` + PolicyName string `bson:"policy_name" json:"policy_name"` + Workflows []WorkflowCIItem `bson:"workflows" json:"workflows"` + Products []ProductCIItem `bson:"products" json:"products"` +} +type ProductCIItem struct { + Name string `bson:"name" json:"name"` + BaseName string `bson:"base_name" json:"base_name"` + CollaborationType config.CollaborationType `bson:"collaboration_type" json:"collaboration_type"` + RecycleDay int64 `bson:"recycle_day" json:"recycle_day"` + Verbs []string `bson:"verbs" json:"verbs"` +} + +type WorkflowCIItem struct { + Name string `bson:"name" json:"name"` + BaseName string `bson:"base_name" json:"base_name"` + CollaborationType config.CollaborationType `bson:"collaboration_type" json:"collaboration_type"` + Verbs []string `bson:"verbs" json:"verbs"` +} + +func (CollaborationInstance) TableName() string { + return "collaboration_instance" +} diff --git a/pkg/microservice/aslan/core/collaboration/repository/models/collaboration_mode.go b/pkg/microservice/aslan/core/collaboration/repository/models/collaboration_mode.go new file mode 100644 index 0000000000000000000000000000000000000000..0f35475e0e8649127175514905746343aaef6e9c --- /dev/null +++ b/pkg/microservice/aslan/core/collaboration/repository/models/collaboration_mode.go @@ -0,0 +1,58 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/collaboration/config" +) + +type CollaborationMode struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + //foreign key:template_product/product_name + ProjectName string `bson:"project_name" json:"project_name"` + CreateTime int64 `bson:"create_time" json:"create_time"` + UpdateTime int64 `bson:"update_time" json:"update_time"` + Name string `bson:"name" json:"name"` + Revision int64 `bson:"revision" json:"revision"` + Members []string `bson:"members" json:"members"` + IsDeleted bool `bson:"is_deleted" json:"is_deleted"` + DeployType string `bson:"deploy_type" json:"deploy_type"` + RecycleDay int64 `bson:"recycle_day" json:"recycle_day"` + Workflows []WorkflowCMItem `bson:"workflows" json:"workflows" ` + Products []ProductCMItem `bson:"products" json:"products"` + CreateBy string `bson:"create_by" json:"create_by"` + UpdateBy string `bson:"update_by" json:"update_by"` +} + +type ProductCMItem struct { + Name string `bson:"name" json:"name"` + CollaborationType config.CollaborationType `bson:"collaboration_type" json:"collaboration_type"` + RecycleDay int64 `bson:"recycle_day" json:"recycle_day"` + Verbs []string `bson:"verbs" json:"verbs"` +} + +type WorkflowCMItem struct { + Name string `bson:"name" json:"name"` + CollaborationType config.CollaborationType `bson:"collaboration_type" json:"collaboration_type"` + Verbs []string `bson:"verbs" json:"verbs"` +} + +func (CollaborationMode) TableName() string { + return "collaboration_mode" +} diff --git a/pkg/microservice/aslan/core/collaboration/repository/mongodb/collaboration_instance.go b/pkg/microservice/aslan/core/collaboration/repository/mongodb/collaboration_instance.go new file mode 100644 index 0000000000000000000000000000000000000000..15a8aee70df6eeb2e141c0516eae5f430b14f5c0 --- /dev/null +++ b/pkg/microservice/aslan/core/collaboration/repository/mongodb/collaboration_instance.go @@ -0,0 +1,202 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "errors" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/collaboration/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type CollaborationInstanceFindOptions struct { + ProjectName string + Name string + UserUID []string +} + +type CollaborationInstanceListOptions struct { + FindOpts []CollaborationInstanceFindOptions +} +type CollaborationInstanceColl struct { + *mongo.Collection + + coll string +} + +func NewCollaborationInstanceColl() *CollaborationInstanceColl { + name := models.CollaborationInstance{}.TableName() + return &CollaborationInstanceColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} +} + +func (c *CollaborationInstanceColl) GetCollectionName() string { + return c.coll +} + +func (c *CollaborationInstanceColl) EnsureIndex(ctx context.Context) error { + mod := []mongo.IndexModel{ + { + Keys: bson.D{ + bson.E{Key: "project_name", Value: 1}, + bson.E{Key: "collaboration_name", Value: 1}, + bson.E{Key: "user_uid", Value: 1}, + }, + Options: options.Index().SetUnique(true), + }, + } + + _, err := c.Indexes().CreateMany(ctx, mod) + + return err +} + +func (c *CollaborationInstanceColl) Find(opt *CollaborationInstanceFindOptions) (*models.CollaborationInstance, error) { + res := &models.CollaborationInstance{} + query := bson.M{} + if len(opt.UserUID) > 0 { + query["user_uid"] = bson.M{"$in": opt.UserUID} + } + if opt.ProjectName != "" { + query["project_name"] = opt.ProjectName + } + err := c.FindOne(context.TODO(), query).Decode(res) + return res, err +} + +func (c *CollaborationInstanceColl) Create(args *models.CollaborationInstance) error { + // avoid panic issue + if args == nil { + return errors.New("nil CollaborationInstance") + } + now := time.Now().Unix() + args.CreateTime = now + args.UpdateTime = now + args.Revision = 1 + _, err := c.InsertOne(context.TODO(), args) + return err +} + +// TODO: Update->BulkUpdate + +func (c *CollaborationInstanceColl) Update(uid string, args *models.CollaborationInstance) error { + if args == nil { + return errors.New("nil CollaborationInstance") + } + + query := bson.M{"collaboration_name": args.CollaborationName, "project_name": args.ProjectName, "user_uid": uid} + change := bson.M{"$set": bson.M{ + "update_time": time.Now().Unix(), + "last_visit_time": args.LastVisitTime, + "workflows": args.Workflows, + "revision": args.Revision, + "products": args.Products, + }} + + _, err := c.UpdateOne(context.TODO(), query, change) + + return err +} + +func (c *CollaborationInstanceColl) ResetRevision(modeName, projectName string) error { + if len(modeName) == 0 || len(projectName) == 0 { + return errors.New("nil modeName or projectName") + } + + query := bson.M{"collaboration_name": modeName, "project_name": projectName} + change := bson.M{"$set": bson.M{ + "update_time": time.Now().Unix(), + "revision": 0, + }} + + _, err := c.UpdateOne(context.TODO(), query, change) + + return err +} + +func (c *CollaborationInstanceColl) BulkCreate(args []*models.CollaborationInstance) error { + if len(args) == 0 { + return nil + } + now := time.Now().Unix() + for _, arg := range args { + arg.CreateTime = now + arg.UpdateTime = now + arg.LastVisitTime = now + } + var ois []interface{} + for _, obj := range args { + ois = append(ois, obj) + } + _, err := c.InsertMany(context.TODO(), ois) + return err +} + +func (c *CollaborationInstanceColl) BulkDelete(opt CollaborationInstanceListOptions) error { + condition := bson.A{} + if len(opt.FindOpts) == 0 { + return nil + } + for _, findOpt := range opt.FindOpts { + condition = append(condition, bson.M{ + "project_name": findOpt.ProjectName, + "user_uid": bson.M{"$in": findOpt.UserUID}, + "collaboration_name": findOpt.Name, + }) + } + filter := bson.D{{"$or", condition}} + _, err := c.DeleteMany(context.TODO(), filter) + return err +} + +func (c *CollaborationInstanceColl) DeleteByProject(projectName string) error { + query := bson.M{} + query["project_name"] = projectName + _, err := c.DeleteMany(context.TODO(), query) + return err +} + +func (c *CollaborationInstanceColl) List(opt *CollaborationInstanceFindOptions) ([]*models.CollaborationInstance, error) { + var ret []*models.CollaborationInstance + query := bson.M{} + if len(opt.UserUID) > 0 { + query["user_uid"] = bson.M{"$in": opt.UserUID} + } + if opt.ProjectName != "" { + query["project_name"] = opt.ProjectName + } + ctx := context.Background() + opts := options.Find() + opts.SetSort(bson.D{{"create_time", -1}}) + cursor, err := c.Collection.Find(ctx, query, opts) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &ret) + if err != nil { + return nil, err + } + + return ret, nil +} diff --git a/pkg/microservice/aslan/core/collaboration/repository/mongodb/collaboration_mode.go b/pkg/microservice/aslan/core/collaboration/repository/mongodb/collaboration_mode.go new file mode 100644 index 0000000000000000000000000000000000000000..41987f12f90b0b47f1d9ac341a9822968a1450b6 --- /dev/null +++ b/pkg/microservice/aslan/core/collaboration/repository/mongodb/collaboration_mode.go @@ -0,0 +1,212 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "errors" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/collaboration/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type CollaborationModeFindOptions struct { + ProjectName string + Name string + Members string + IsDeleted bool +} + +type CollaborationModeListOptions struct { + Projects []string + Name string + Members []string + IsDeleted bool +} + +type CollaborationModeColl struct { + *mongo.Collection + + coll string +} + +func NewCollaborationModeColl() *CollaborationModeColl { + name := models.CollaborationMode{}.TableName() + return &CollaborationModeColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} +} + +func (c *CollaborationModeColl) GetCollectionName() string { + return c.coll +} + +func (c *CollaborationModeColl) EnsureIndex(ctx context.Context) error { + mod := []mongo.IndexModel{ + { + Keys: bson.D{ + bson.E{Key: "project_name", Value: 1}, + bson.E{Key: "members", Value: 1}, + bson.E{Key: "is_deleted", Value: 1}, + }, + Options: options.Index().SetUnique(false), + }, + { + Keys: bson.D{ + bson.E{Key: "project_name", Value: 1}, + bson.E{Key: "name", Value: 1}, + bson.E{Key: "is_deleted", Value: 1}, + }, + Options: options.Index().SetUnique(false), + }, + } + + _, err := c.Indexes().CreateMany(ctx, mod) + + return err +} + +func (c *CollaborationModeColl) Update(username string, args *models.CollaborationMode) error { + if args == nil { + return errors.New("nil CollaborationMode") + } + res, err := c.Find(&CollaborationModeFindOptions{ + Name: args.Name, + ProjectName: args.ProjectName, + IsDeleted: false, + }) + if err == nil { + query := bson.M{"name": args.Name, "project_name": args.ProjectName, "is_deleted": false} + change := bson.M{"$set": bson.M{ + "update_time": time.Now().Unix(), + "update_by": username, + "members": args.Members, + "workflows": args.Workflows, + "recycle_day": args.RecycleDay, + "revision": res.Revision + 1, + "products": args.Products, + }} + + _, err = c.UpdateOne(context.TODO(), query, change) + + return err + } + return nil +} + +func (c *CollaborationModeColl) Create(userName string, args *models.CollaborationMode) error { + // avoid panic issue + if args == nil { + return errors.New("nil CollaborationMode") + } + _, err := c.Find(&CollaborationModeFindOptions{ + Name: args.Name, + ProjectName: args.ProjectName, + IsDeleted: false, + }) + + if err == nil { + return errors.New("the collaborationMode is already exist in this project") + } + now := time.Now().Unix() + args.CreateBy = userName + args.UpdateBy = userName + args.CreateTime = now + args.UpdateTime = now + args.Revision = 1 + args.IsDeleted = false + _, err = c.InsertOne(context.TODO(), args) + return err +} + +func (c *CollaborationModeColl) DeleteByProject(project string) error { + query := bson.M{"project_name": project} + _, err := c.DeleteMany(context.TODO(), query) + return err +} + +func (c *CollaborationModeColl) Delete(username, projectName, name string) error { + _, err := c.Find(&CollaborationModeFindOptions{ + Name: name, + ProjectName: projectName, + IsDeleted: false, + }) + if err == nil { + deleteQuery := bson.M{"name": name, "project_name": projectName, "is_deleted": true} + _, err = c.DeleteOne(context.TODO(), deleteQuery) + if err != nil { + return err + } + } + query := bson.M{"name": name, "project_name": projectName, "is_deleted": false} + change := bson.M{"$set": bson.M{ + "delete_time": time.Now().Unix(), + "delete_by": username, + "is_deleted": true, + }} + _, err = c.UpdateOne(context.TODO(), query, change) + return err +} + +func (c *CollaborationModeColl) Find(opt *CollaborationModeFindOptions) (*models.CollaborationMode, error) { + res := &models.CollaborationMode{} + query := bson.M{} + if opt.Name != "" { + query["name"] = opt.Name + } + if opt.ProjectName != "" { + query["project_name"] = opt.ProjectName + } + query["is_deleted"] = false + err := c.FindOne(context.TODO(), query).Decode(res) + return res, err +} + +func (c *CollaborationModeColl) List(opt *CollaborationModeListOptions) ([]*models.CollaborationMode, error) { + var ret []*models.CollaborationMode + query := bson.M{} + if opt.Name != "" { + query["name"] = opt.Name + } + if len(opt.Projects) > 0 { + query["project_name"] = bson.M{"$in": opt.Projects} + } + if len(opt.Members) > 0 { + opt.Members = append(opt.Members, "*") + query["members"] = bson.M{"$in": opt.Members} + } + query["is_deleted"] = false + + ctx := context.Background() + opts := options.Find() + opts.SetSort(bson.D{{"create_time", -1}}) + cursor, err := c.Collection.Find(ctx, query, opts) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &ret) + if err != nil { + return nil, err + } + + return ret, nil +} diff --git a/pkg/microservice/aslan/core/collaboration/service/collaboration_instance.go b/pkg/microservice/aslan/core/collaboration/service/collaboration_instance.go new file mode 100644 index 0000000000000000000000000000000000000000..20fa0859cd33886402e6120adc546763cd85ba60 --- /dev/null +++ b/pkg/microservice/aslan/core/collaboration/service/collaboration_instance.go @@ -0,0 +1,1374 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "fmt" + "reflect" + "time" + + "go.mongodb.org/mongo-driver/mongo" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/collaboration/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/collaboration/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/collaboration/repository/mongodb" + models2 "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + templatemodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + service2 "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" + config2 "github.com/koderover/zadig/pkg/microservice/aslan/core/label/config" + mongodb2 "github.com/koderover/zadig/pkg/microservice/aslan/core/label/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/label/service" + workflowservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/client/policy" + "github.com/koderover/zadig/pkg/types" +) + +type GetCollaborationUpdateResp struct { + UpdateInstance []models.CollaborationInstance `json:"update_instance"` + Update []UpdateItem `json:"update"` + New []models.CollaborationMode `json:"new"` + Delete []models.CollaborationInstance `json:"delete"` +} + +type UpdateItem struct { + CollaborationMode string `json:"collaboration_mode"` + PolicyName string `json:"policy_name"` + DeployType string `json:"deploy_type"` + DeleteSpec DeleteSpec `json:"delete_spec"` + UpdateSpec UpdateSpec `json:"update_spec"` + NewSpec NewSpec `json:"new_spec"` +} + +type DeleteSpec struct { + Workflows []models.WorkflowCIItem `json:"workflows"` + Products []models.ProductCIItem `json:"products"` +} + +type UpdateSpec struct { + Workflows []UpdateWorkflowItem `json:"workflows"` + Products []UpdateProductItem `json:"products"` +} + +type NewSpec struct { + Workflows []models.WorkflowCMItem `json:"workflows"` + Products []models.ProductCMItem `json:"products"` +} + +type UpdateWorkflowItem struct { + Old models.WorkflowCIItem `json:"old"` + New models.WorkflowCMItem `json:"new"` +} + +type UpdateProductItem struct { + Old models.ProductCIItem `json:"old"` + New models.ProductCMItem `json:"new"` +} + +type Workflow struct { + CollaborationType config.CollaborationType `json:"collaboration_type"` + BaseName string `json:"base_name"` + CollaborationMode string `json:"collaboration_mode"` + Name string `json:"name"` + Description string `json:"description"` +} + +type Product struct { + CollaborationType config.CollaborationType `json:"collaboration_type"` + BaseName string `json:"base_name"` + CollaborationMode string `json:"collaboration_mode"` + Name string `json:"name"` + DeployType string `json:"deploy_type"` + Vars []*templatemodels.RenderKV `json:"vars"` + DefaultValues string `json:"defaultValues,omitempty"` + ChartValues []*commonservice.RenderChartArg `json:"chartValues,omitempty"` +} + +type GetCollaborationNewResp struct { + Code int64 `json:"code"` + Workflow []*Workflow `json:"workflow"` + Product []*Product `json:"product"` + IfSync bool `json:"ifSync"` +} + +type GetCollaborationDeleteResp struct { + Workflows []string + Products []string +} + +func getUpdateWorkflowDiff(cmwMap map[string]models.WorkflowCMItem, ciwMap map[string]models.WorkflowCIItem) ( + []UpdateWorkflowItem, []models.WorkflowCMItem, []models.WorkflowCIItem) { + var updateWorkflowItems []UpdateWorkflowItem + var newWorkflowItems []models.WorkflowCMItem + var deleteWorkflowItems []models.WorkflowCIItem + for name, cm := range cmwMap { + if ci, ok := ciwMap[name]; ok { + if !reflect.DeepEqual(cm.Verbs, ci.Verbs) || cm.CollaborationType != ci.CollaborationType { + updateWorkflowItems = append(updateWorkflowItems, UpdateWorkflowItem{ + Old: ci, + New: cm, + }) + } + } else { + newWorkflowItems = append(newWorkflowItems, cm) + } + } + for name, ci := range ciwMap { + if _, ok := cmwMap[name]; !ok { + deleteWorkflowItems = append(deleteWorkflowItems, ci) + } + } + return updateWorkflowItems, newWorkflowItems, deleteWorkflowItems +} + +func getUpdateProductDiff(cmpMap map[string]models.ProductCMItem, cipMap map[string]models.ProductCIItem) ( + []UpdateProductItem, []models.ProductCMItem, []models.ProductCIItem) { + var updateProductItems []UpdateProductItem + var newProductItems []models.ProductCMItem + var deleteProductItems []models.ProductCIItem + for name, cm := range cmpMap { + if ci, ok := cipMap[name]; ok { + if !reflect.DeepEqual(cm.Verbs, ci.Verbs) || cm.CollaborationType != ci.CollaborationType { + updateProductItems = append(updateProductItems, UpdateProductItem{ + Old: ci, + New: cm, + }) + } + } else { + newProductItems = append(newProductItems, cm) + } + } + for name, ci := range cipMap { + if _, ok := cmpMap[name]; !ok { + deleteProductItems = append(deleteProductItems, ci) + } + } + return updateProductItems, newProductItems, deleteProductItems +} + +func getUpdateDiff(cm *models.CollaborationMode, ci *models.CollaborationInstance) UpdateItem { + cmwMap := make(map[string]models.WorkflowCMItem) + for _, workflow := range cm.Workflows { + cmwMap[workflow.Name] = workflow + } + ciwMap := make(map[string]models.WorkflowCIItem) + for _, workflow := range ci.Workflows { + ciwMap[workflow.BaseName] = workflow + } + updateWorkflowItems, newWorkflowItems, deleteWorkflowItems := getUpdateWorkflowDiff(cmwMap, ciwMap) + cmpMap := make(map[string]models.ProductCMItem) + for _, product := range cm.Products { + cmpMap[product.Name] = product + } + cipMap := make(map[string]models.ProductCIItem) + for _, product := range ci.Products { + cipMap[product.BaseName] = product + } + updateProductItems, newProductItems, deleteProductItems := getUpdateProductDiff(cmpMap, cipMap) + return UpdateItem{ + CollaborationMode: cm.Name, + PolicyName: ci.PolicyName, + DeployType: cm.DeployType, + NewSpec: NewSpec{ + Workflows: newWorkflowItems, + Products: newProductItems, + }, + UpdateSpec: UpdateSpec{ + Workflows: updateWorkflowItems, + Products: updateProductItems, + }, + DeleteSpec: DeleteSpec{ + Workflows: deleteWorkflowItems, + Products: deleteProductItems, + }, + } +} + +func buildPolicyName(projectName, mode, identityType, userName string) string { + return projectName + "-" + mode + "-" + identityType + "-" + userName +} + +func genCollaborationInstance(mode models.CollaborationMode, projectName, uid, identityType, userName string) *models.CollaborationInstance { + var workflows []models.WorkflowCIItem + for _, workflow := range mode.Workflows { + name := workflow.Name + if workflow.CollaborationType == config.CollaborationNew { + name = buildName(workflow.Name, mode.Name, identityType, userName) + } + workflows = append(workflows, models.WorkflowCIItem{ + Name: name, + BaseName: workflow.Name, + Verbs: workflow.Verbs, + CollaborationType: workflow.CollaborationType, + }) + } + var products []models.ProductCIItem + for _, product := range mode.Products { + name := product.Name + if product.CollaborationType == config.CollaborationNew { + name = buildName(product.Name, mode.Name, identityType, userName) + } + products = append(products, models.ProductCIItem{ + Name: name, + BaseName: product.Name, + CollaborationType: product.CollaborationType, + Verbs: product.Verbs, + }) + } + return &models.CollaborationInstance{ + ProjectName: mode.ProjectName, + CollaborationName: mode.Name, + UserUID: uid, + PolicyName: buildPolicyName(projectName, mode.Name, identityType, userName), + Revision: mode.Revision, + RecycleDay: mode.RecycleDay, + Workflows: workflows, + Products: products, + LastVisitTime: time.Now().Unix(), + } +} + +func getDiff(cmMap map[string]*models.CollaborationMode, ciMap map[string]*models.CollaborationInstance, projectName, + uid, identityType, userName string) (*GetCollaborationUpdateResp, error) { + var updateItems []UpdateItem + var updateInstance []models.CollaborationInstance + var newItems []models.CollaborationMode + var deleteItems []models.CollaborationInstance + for name, cm := range cmMap { + if ci, ok := ciMap[name]; ok { + if cm.Revision < ci.Revision { + return nil, fmt.Errorf("CollaborationMode:%s revision error", name) + } else if cm.Revision > ci.Revision { + updateItems = append(updateItems, getUpdateDiff(cm, ci)) + instance := genCollaborationInstance(*cm, projectName, uid, identityType, userName) + updateInstance = append(updateInstance, *instance) + } + } else { + newItems = append(newItems, *cm) + } + } + for name, ci := range ciMap { + if _, ok := cmMap[name]; !ok { + deleteItems = append(deleteItems, *ci) + } + } + return &GetCollaborationUpdateResp{ + UpdateInstance: updateInstance, + Update: updateItems, + New: newItems, + Delete: deleteItems, + }, nil +} + +func updateVisitTime(uid string, cis []*models.CollaborationInstance, logger *zap.SugaredLogger) error { + for _, instance := range cis { + instance.LastVisitTime = time.Now().Unix() + err := mongodb.NewCollaborationInstanceColl().Update(uid, instance) + if err != nil { + logger.Errorf("syncInstance Update error, error msg:%s", err) + return err + } + } + return nil +} + +func GetCollaborationUpdate(projectName, uid, identityType, userName string, logger *zap.SugaredLogger) (*GetCollaborationUpdateResp, error) { + collaborations, err := mongodb.NewCollaborationModeColl().List(&mongodb.CollaborationModeListOptions{ + Projects: []string{projectName}, + Members: []string{uid}, + }) + if err != nil { + logger.Errorf("GetCollaborationUpdate error, err msg:%s", err) + return nil, err + } + cmMap := make(map[string]*models.CollaborationMode) + for _, collaboration := range collaborations { + cmMap[collaboration.Name] = collaboration + } + collaborationInstances, err := mongodb.NewCollaborationInstanceColl().List(&mongodb.CollaborationInstanceFindOptions{ + ProjectName: projectName, + UserUID: []string{uid}, + }) + if err != nil { + logger.Errorf("GetCollaborationInstance error, err msg:%s", err) + return nil, err + } + ciMap := make(map[string]*models.CollaborationInstance) + for _, instance := range collaborationInstances { + ciMap[instance.CollaborationName] = instance + } + resp, err := getDiff(cmMap, ciMap, projectName, uid, identityType, userName) + if err != nil { + logger.Errorf("GetCollaborationUpdate error, err msg:%s", err) + return nil, err + } + err = updateVisitTime(uid, collaborationInstances, logger) + if err != nil { + logger.Errorf("GetCollaborationUpdate updateVisitTime error, err msg:%s", err) + return nil, err + } + return resp, nil +} + +func buildName(baseName, modeName, identityType, userName string) string { + return modeName + "-" + baseName + "-" + identityType + "-" + userName +} + +func syncInstance(updateResp *GetCollaborationUpdateResp, projectName, identityType, userName, uid string, + logger *zap.SugaredLogger) error { + var instances []*models.CollaborationInstance + modeInstanceMap := make(map[string]*models.CollaborationInstance) + for _, mode := range updateResp.New { + instance := genCollaborationInstance(mode, projectName, uid, identityType, userName) + instances = append(instances, instance) + modeInstanceMap[mode.Name] = instance + } + if len(instances) > 0 { + err := mongodb.NewCollaborationInstanceColl().BulkCreate(instances) + if err != nil { + logger.Errorf("syncInstance BulkCreate error, error msg:%s", err) + return err + } + } + for _, instance := range updateResp.UpdateInstance { + err := mongodb.NewCollaborationInstanceColl().Update(uid, &instance) + if err != nil { + logger.Errorf("syncInstance Update error, error msg:%s", err) + return err + } + } + var findOpts []mongodb.CollaborationInstanceFindOptions + for _, instance := range updateResp.Delete { + findOpts = append(findOpts, mongodb.CollaborationInstanceFindOptions{ + Name: instance.CollaborationName, + ProjectName: instance.ProjectName, + UserUID: []string{instance.UserUID}, + }) + } + return mongodb.NewCollaborationInstanceColl().BulkDelete(mongodb.CollaborationInstanceListOptions{ + FindOpts: findOpts, + }) +} + +func buildPolicyDescription(mode, userName string) string { + return mode + " " + userName + " 的权限" +} + +func buildPolicybindingName(uid, policyName, projectName string) string { + return uid + "-" + policyName + "-" + projectName +} + +func syncPolicy(updateResp *GetCollaborationUpdateResp, projectName, identityType, userName, uid string, + logger *zap.SugaredLogger) error { + var policies []*types.Policy + var policyBindings []*policy.PolicyBinding + for _, mode := range updateResp.New { + var rules []*types.Rule + + policyName := buildPolicyName(projectName, mode.Name, identityType, userName) + + for _, workflow := range mode.Workflows { + rules = append(rules, &types.Rule{ + Verbs: workflow.Verbs, + Kind: "resource", + Resources: []string{string(config2.ResourceTypeWorkflow)}, + MatchAttributes: []types.MatchAttribute{ + { + Key: "policy", + Value: buildLabelValue(projectName, mode.Name, identityType, userName, string(config2.ResourceTypeWorkflow), workflow.Name), + }, + }, + }) + } + for _, product := range mode.Products { + rules = append(rules, &types.Rule{ + Verbs: product.Verbs, + Kind: "resource", + Resources: []string{string(config2.ResourceTypeEnvironment)}, + MatchAttributes: []types.MatchAttribute{ + { + Key: "policy", + Value: buildLabelValue(projectName, mode.Name, identityType, userName, string(config2.ResourceTypeEnvironment), product.Name), + }, + }, + }) + } + policies = append(policies, &types.Policy{ + Name: policyName, + UpdateTime: time.Now().Unix(), + Description: buildPolicyDescription(mode.Name, userName), + Rules: rules, + }) + policyBindings = append(policyBindings, &policy.PolicyBinding{ + Name: buildPolicybindingName(uid, policyName, projectName), + UID: uid, + Policy: policyName, + Preset: false, + Type: setting.ResourceTypeSystem, + }) + } + if len(policies) > 0 { + err := policy.NewDefault().CreatePolicies(projectName, policy.CreatePoliciesArgs{ + Policies: policies, + }) + if err != nil { + logger.Errorf("syncPolicy error, error msg:%s", err) + return err + } + } + if len(policyBindings) > 0 { + err := policy.NewDefault().CreatePolicyBinding(projectName, policyBindings) + if err != nil { + logger.Errorf("syncPolicyBindings error, error msg:%s", err) + return err + } + } + var updatePolicies []*types.Policy + for _, instance := range updateResp.UpdateInstance { + var rules []*types.Rule + for _, workflow := range instance.Workflows { + rules = append(rules, &types.Rule{ + Verbs: workflow.Verbs, + Kind: "resource", + Resources: []string{string(config2.ResourceTypeWorkflow)}, + MatchAttributes: []types.MatchAttribute{ + { + Key: "policy", + Value: buildLabelValue(projectName, instance.CollaborationName, identityType, userName, string(config2.ResourceTypeWorkflow), workflow.BaseName), + }, + }, + }) + } + for _, product := range instance.Products { + rules = append(rules, &types.Rule{ + Verbs: product.Verbs, + Kind: "resource", + Resources: []string{string(config2.ResourceTypeEnvironment)}, + MatchAttributes: []types.MatchAttribute{ + { + Key: "policy", + Value: buildLabelValue(projectName, instance.CollaborationName, identityType, userName, string(config2.ResourceTypeEnvironment), product.BaseName), + }, + }, + }) + } + updatePolicies = append(updatePolicies, &types.Policy{ + Name: instance.PolicyName, + Description: buildPolicyDescription(instance.CollaborationName, userName), + UpdateTime: time.Now().Unix(), + Rules: rules, + }) + } + for _, updatePolicy := range updatePolicies { + err := policy.NewDefault().UpdatePolicy(projectName, updatePolicy) + if err != nil { + return err + } + } + var deletePolicies []string + var deletePolicybindings []string + for _, instance := range updateResp.Delete { + deletePolicies = append(deletePolicies, instance.PolicyName) + deletePolicybindings = append(deletePolicybindings, buildPolicybindingName(uid, instance.PolicyName, projectName)) + } + if len(deletePolicies) > 0 { + err := policy.NewDefault().DeletePolicies(projectName, policy.DeletePoliciesArgs{ + Names: deletePolicies, + }) + if err != nil { + return err + } + } + if len(deletePolicybindings) > 0 { + err := policy.NewDefault().DeletePolicyBindings(deletePolicybindings, projectName) + if err != nil { + return err + } + } + return nil +} + +func buildLabelValue(projectName, mode, identityType, userName, resourceType, resourceName string) string { + return projectName + "-" + mode + "-" + identityType + "-" + userName + "-" + resourceType + "-" + resourceName +} + +func syncLabel(updateResp *GetCollaborationUpdateResp, projectName, identityType, userName string, logger *zap.SugaredLogger) error { + var labels []mongodb2.Label + var deleteLabels []mongodb2.Label + var newBindings []*mongodb2.LabelBinding + var deleteBindings []*mongodb2.LabelBinding + for _, mode := range updateResp.New { + for _, workflow := range mode.Workflows { + labels = append(labels, mongodb2.Label{ + Key: "policy", + Value: buildLabelValue(projectName, mode.Name, identityType, userName, + string(config2.ResourceTypeWorkflow), workflow.Name), + Type: setting.ResourceTypeSystem, + ProjectName: projectName, + }) + } + for _, product := range mode.Products { + labels = append(labels, mongodb2.Label{ + Key: "policy", + Value: buildLabelValue(projectName, mode.Name, identityType, userName, + string(config2.ResourceTypeEnvironment), product.Name), + Type: setting.ResourceTypeSystem, + ProjectName: projectName, + }) + } + } + for _, item := range updateResp.Update { + for _, workflow := range item.NewSpec.Workflows { + labels = append(labels, mongodb2.Label{ + Key: "policy", + Value: buildLabelValue(projectName, item.CollaborationMode, identityType, userName, + string(config2.ResourceTypeWorkflow), workflow.Name), + Type: setting.ResourceTypeSystem, + ProjectName: projectName, + }) + } + for _, product := range item.NewSpec.Products { + labels = append(labels, mongodb2.Label{ + Key: "policy", + Value: buildLabelValue(projectName, item.CollaborationMode, identityType, userName, + string(config2.ResourceTypeEnvironment), product.Name), + Type: setting.ResourceTypeSystem, + ProjectName: projectName, + }) + } + for _, workflow := range item.DeleteSpec.Workflows { + deleteLabels = append(deleteLabels, mongodb2.Label{ + Key: "policy", + Value: buildLabelValue(projectName, item.CollaborationMode, identityType, userName, + string(config2.ResourceTypeWorkflow), workflow.BaseName), + Type: setting.ResourceTypeSystem, + ProjectName: projectName, + }) + } + for _, product := range item.DeleteSpec.Products { + deleteLabels = append(deleteLabels, mongodb2.Label{ + Key: "policy", + Value: buildLabelValue(projectName, item.CollaborationMode, identityType, userName, + string(config2.ResourceTypeEnvironment), product.BaseName), + Type: setting.ResourceTypeSystem, + ProjectName: projectName, + }) + } + + } + for _, instance := range updateResp.Delete { + for _, workflow := range instance.Workflows { + deleteLabels = append(deleteLabels, mongodb2.Label{ + Key: "policy", + Value: buildLabelValue(projectName, instance.CollaborationName, identityType, userName, + string(config2.ResourceTypeWorkflow), workflow.BaseName), + Type: setting.ResourceTypeSystem, + }) + } + for _, product := range instance.Products { + deleteLabels = append(deleteLabels, mongodb2.Label{ + Key: "policy", + Value: buildLabelValue(projectName, instance.CollaborationName, identityType, userName, + string(config2.ResourceTypeEnvironment), product.BaseName), + Type: setting.ResourceTypeSystem, + }) + } + } + var err error + if len(labels) > 0 { + _, err := service.CreateLabels(&service.CreateLabelsArgs{ + Labels: labels, + }, userName) + + if err != nil { + logger.Errorf("create labels error, error msg:%s", err) + return err + } + } + + if len(deleteLabels) > 0 { + resp, err := service.ListLabels(&service.ListLabelsArgs{ + Labels: deleteLabels, + }) + if err != nil { + return err + } + if len(resp.Labels) != len(deleteLabels) { + return fmt.Errorf("deletelabels not exist,%v:%v", resp.Labels, deleteLabels) + } + var ids []string + for _, label := range resp.Labels { + ids = append(ids, label.ID.Hex()) + } + err = service.DeleteLabels(ids, true, userName, logger) + if err != nil { + logger.Errorf("delete labels error, error msg:%s", err) + return err + } + } + + for _, item := range updateResp.Update { + for _, workflow := range item.UpdateSpec.Workflows { + labels = append(labels, mongodb2.Label{ + Key: "policy", + Value: buildLabelValue(projectName, item.CollaborationMode, identityType, userName, string(config2.ResourceTypeWorkflow), workflow.New.Name), + Type: setting.ResourceTypeSystem, + }) + } + for _, product := range item.UpdateSpec.Products { + labels = append(labels, mongodb2.Label{ + Key: "policy", + Value: buildLabelValue(projectName, item.CollaborationMode, identityType, userName, string(config2.ResourceTypeEnvironment), product.New.Name), + Type: setting.ResourceTypeSystem, + }) + } + + } + labelIdMap := make(map[string]string) + if len(labels) > 0 { + labelListResp, err := service.ListLabels(&service.ListLabelsArgs{ + Labels: labels, + }) + if err != nil { + return err + } + if labelListResp == nil { + return fmt.Errorf("label not exist %v", labels) + } + if len(labelListResp.Labels) != len(labels) { + return fmt.Errorf("label not exist %v:%v", labels, labelListResp.Labels) + } + + for _, label := range labelListResp.Labels { + labelIdMap[service.BuildLabelString(label.Key, label.Value)] = label.ID.Hex() + } + + } + + for _, mode := range updateResp.New { + for _, workflow := range mode.Workflows { + labelValue := buildLabelValue(projectName, mode.Name, identityType, userName, string(config2.ResourceTypeWorkflow), workflow.Name) + labelId, ok := labelIdMap[service.BuildLabelString("policy", labelValue)] + if !ok { + return fmt.Errorf("label:%s not exist", labelValue) + } + name := workflow.Name + if workflow.CollaborationType == config.CollaborationNew { + name = buildName(workflow.Name, mode.Name, identityType, userName) + } + newBindings = append(newBindings, &mongodb2.LabelBinding{ + LabelID: labelId, + Resource: mongodb2.Resource{ + Name: name, + ProjectName: projectName, + Type: string(config2.ResourceTypeWorkflow), + }, + }) + } + for _, product := range mode.Products { + labelValue := buildLabelValue(projectName, mode.Name, identityType, userName, string(config2.ResourceTypeEnvironment), product.Name) + labelId, ok := labelIdMap[service.BuildLabelString("policy", labelValue)] + if !ok { + return fmt.Errorf("label:%s not exist", labelValue) + } + name := product.Name + if product.CollaborationType == config.CollaborationNew { + name = buildName(product.Name, mode.Name, identityType, userName) + } + newBindings = append(newBindings, &mongodb2.LabelBinding{ + LabelID: labelId, + Resource: mongodb2.Resource{ + Name: name, + ProjectName: projectName, + Type: string(config2.ResourceTypeEnvironment), + }, + }) + } + } + for _, item := range updateResp.Update { + for _, workflow := range item.NewSpec.Workflows { + labelValue := buildLabelValue(projectName, item.CollaborationMode, identityType, userName, string(config2.ResourceTypeWorkflow), workflow.Name) + labelId, ok := labelIdMap[service.BuildLabelString("policy", labelValue)] + if !ok { + return fmt.Errorf("label:%s not exist", labelValue) + } + name := workflow.Name + if workflow.CollaborationType == config.CollaborationNew { + name = buildName(workflow.Name, item.CollaborationMode, identityType, userName) + } + newBindings = append(newBindings, &mongodb2.LabelBinding{ + LabelID: labelId, + Resource: mongodb2.Resource{ + Name: name, + Type: string(config2.ResourceTypeWorkflow), + ProjectName: projectName, + }, + }) + } + for _, product := range item.NewSpec.Products { + labelValue := buildLabelValue(projectName, item.CollaborationMode, identityType, userName, string(config2.ResourceTypeEnvironment), product.Name) + labelId, ok := labelIdMap[service.BuildLabelString("policy", labelValue)] + if !ok { + return fmt.Errorf("label:%s not exist", labelValue) + } + name := product.Name + if product.CollaborationType == config.CollaborationNew { + name = buildName(product.Name, item.CollaborationMode, identityType, userName) + } + newBindings = append(newBindings, &mongodb2.LabelBinding{ + LabelID: labelId, + Resource: mongodb2.Resource{ + Name: name, + Type: string(config2.ResourceTypeEnvironment), + ProjectName: projectName, + }, + }) + } + for _, workflow := range item.UpdateSpec.Workflows { + if workflow.Old.CollaborationType == config.CollaborationShare && workflow.New.CollaborationType == config.CollaborationNew { + labelValue := buildLabelValue(projectName, item.CollaborationMode, identityType, userName, string(config2.ResourceTypeWorkflow), workflow.New.Name) + labelId, ok := labelIdMap[service.BuildLabelString("policy", labelValue)] + if !ok { + return fmt.Errorf("label:%s not exist", labelValue) + } + newBindings = append(newBindings, &mongodb2.LabelBinding{ + LabelID: labelId, + Resource: mongodb2.Resource{ + Name: buildName(workflow.New.Name, item.CollaborationMode, identityType, userName), + Type: string(config2.ResourceTypeWorkflow), + ProjectName: projectName, + }, + }) + deleteBindings = append(deleteBindings, &mongodb2.LabelBinding{ + LabelID: labelId, + Resource: mongodb2.Resource{ + Name: workflow.Old.BaseName, + Type: string(config2.ResourceTypeWorkflow), + ProjectName: projectName, + }, + }) + } + if workflow.Old.CollaborationType == config.CollaborationNew && workflow.New.CollaborationType == config.CollaborationShare { + labelValue := buildLabelValue(projectName, item.CollaborationMode, identityType, userName, string(config2.ResourceTypeWorkflow), workflow.New.Name) + labelId, ok := labelIdMap[service.BuildLabelString("policy", labelValue)] + if !ok { + return fmt.Errorf("label:%s not exist", labelValue) + } + newBindings = append(newBindings, &mongodb2.LabelBinding{ + LabelID: labelId, + Resource: mongodb2.Resource{ + Name: workflow.New.Name, + Type: string(config2.ResourceTypeWorkflow), + ProjectName: projectName, + }, + }) + deleteBindings = append(deleteBindings, &mongodb2.LabelBinding{ + LabelID: labelId, + Resource: mongodb2.Resource{ + Name: workflow.Old.Name, + Type: string(config2.ResourceTypeWorkflow), + ProjectName: projectName, + }, + }) + } + } + for _, product := range item.UpdateSpec.Products { + if product.Old.CollaborationType == config.CollaborationShare && product.New.CollaborationType == config.CollaborationNew { + labelValue := buildLabelValue(projectName, item.CollaborationMode, identityType, userName, string(config2.ResourceTypeEnvironment), product.New.Name) + labelId, ok := labelIdMap[service.BuildLabelString("policy", labelValue)] + if !ok { + return fmt.Errorf("label:%s not exist", labelValue) + } + newBindings = append(newBindings, &mongodb2.LabelBinding{ + LabelID: labelId, + Resource: mongodb2.Resource{ + Name: buildName(product.New.Name, item.CollaborationMode, identityType, userName), + Type: string(config2.ResourceTypeEnvironment), + ProjectName: projectName, + }, + }) + deleteBindings = append(deleteBindings, &mongodb2.LabelBinding{ + LabelID: labelId, + Resource: mongodb2.Resource{ + Name: product.Old.BaseName, + Type: string(config2.ResourceTypeEnvironment), + ProjectName: projectName, + }, + }) + } + if product.Old.CollaborationType == config.CollaborationNew && product.New.CollaborationType == config.CollaborationShare { + labelValue := buildLabelValue(projectName, item.CollaborationMode, identityType, userName, string(config2.ResourceTypeEnvironment), product.New.Name) + labelId, ok := labelIdMap[service.BuildLabelString("policy", labelValue)] + if !ok { + return fmt.Errorf("label:%s not exist", labelValue) + } + newBindings = append(newBindings, &mongodb2.LabelBinding{ + LabelID: labelId, + Resource: mongodb2.Resource{ + Name: product.New.Name, + Type: string(config2.ResourceTypeEnvironment), + ProjectName: projectName, + }, + }) + deleteBindings = append(deleteBindings, &mongodb2.LabelBinding{ + LabelID: labelId, + Resource: mongodb2.Resource{ + Name: product.Old.Name, + Type: string(config2.ResourceTypeEnvironment), + ProjectName: projectName, + }, + }) + } + } + } + + if len(newBindings) > 0 { + err = service.CreateLabelBindings(&service.CreateLabelBindingsArgs{ + LabelBindings: newBindings, + }, userName, logger) + if err != nil { + logger.Errorf("create labelbindings error:%s", err) + return err + } + } + if len(deleteBindings) > 0 { + logger.Infof("start syncLabel DeleteLabelBindings:%v user:%s", deleteBindings, userName) + err = service.DeleteLabelBindings(&service.DeleteLabelBindingsArgs{ + LabelBindings: deleteBindings, + }, userName, logger) + if err != nil { + logger.Errorf("delete labelbindings error:%s", err) + return err + } + } + return nil +} + +func syncResource(products *SyncCollaborationInstanceArgs, updateResp *GetCollaborationUpdateResp, projectName, identityType, userName, requestID string, + logger *zap.SugaredLogger) error { + err := syncNewResource(products, updateResp, projectName, identityType, userName, requestID, logger) + if err != nil { + return err + } + err = syncDeleteResource(updateResp, userName, projectName, requestID, logger) + if err != nil { + return err + } + return nil +} + +func syncDeleteResource(updateResp *GetCollaborationUpdateResp, username, projectName, requestID string, + log *zap.SugaredLogger) (err error) { + deleteResp := getCollaborationDelete(updateResp) + for _, product := range deleteResp.Products { + err := service2.DeleteProduct(username, product, projectName, requestID, true, log) + if err != nil && err != mongo.ErrNoDocuments { + log.Errorf("delete product err:%v", err) + return err + } + } + for _, workflow := range deleteResp.Workflows { + err := commonservice.DeleteWorkflow(workflow, requestID, false, log) + if err != nil && err != mongo.ErrNoDocuments { + log.Errorf("delete workflow err:%v", err) + return err + } + } + return nil +} + +func syncNewResource(products *SyncCollaborationInstanceArgs, updateResp *GetCollaborationUpdateResp, projectName, identityType, userName, requestID string, + logger *zap.SugaredLogger) error { + newResp, err := getCollaborationNew(updateResp, projectName, identityType, userName, logger) + if err != nil { + return err + } + if newResp.Workflow == nil && newResp.Product == nil { + return nil + } + var newWorkflows []workflowservice.WorkflowCopyItem + for _, workflow := range newResp.Workflow { + if workflow.CollaborationType == config.CollaborationNew { + newWorkflows = append(newWorkflows, workflowservice.WorkflowCopyItem{ + ProjectName: projectName, + Old: workflow.BaseName, + New: workflow.Name, + BaseName: workflow.BaseName, + }) + } + } + if len(newWorkflows) > 0 { + err = workflowservice.BulkCopyWorkflow(workflowservice.BulkCopyWorkflowArgs{ + Items: newWorkflows, + }, userName, logger) + if err != nil { + return err + } + } + + productMap := make(map[string]Product) + for _, product := range products.Products { + productMap[product.BaseName] = product + } + var yamlProductItems []service2.YamlProductItem + var helmProductArgs []service2.HelmProductItem + for _, product := range newResp.Product { + if productArg, ok := productMap[product.BaseName]; ok { + if productArg.DeployType == string(setting.HelmDeployType) { + helmProductArgs = append(helmProductArgs, service2.HelmProductItem{ + OldName: product.BaseName, + NewName: product.Name, + BaseName: product.BaseName, + DefaultValues: productArg.DefaultValues, + ChartValues: productArg.ChartValues, + }) + } + if productArg.DeployType == string(setting.K8SDeployType) { + yamlProductItems = append(yamlProductItems, service2.YamlProductItem{ + OldName: product.BaseName, + NewName: product.Name, + BaseName: product.BaseName, + Vars: productArg.Vars, + }) + } + } + } + if len(yamlProductItems) > 0 { + err = service2.BulkCopyYamlProduct(projectName, userName, requestID, service2.CopyYamlProductArg{ + Items: yamlProductItems, + }, logger) + if err != nil { + return err + } + } + if len(helmProductArgs) > 0 { + err = service2.BulkCopyHelmProduct(projectName, userName, requestID, service2.CopyHelmProductArg{ + Items: helmProductArgs, + }, logger) + if err != nil { + return err + } + } + return nil +} + +type SyncCollaborationInstanceArgs struct { + Products []Product `json:"products"` +} + +func SyncCollaborationInstance(products *SyncCollaborationInstanceArgs, projectName, uid, identityType, userName, requestID string, logger *zap.SugaredLogger) error { + updateResp, err := GetCollaborationUpdate(projectName, uid, identityType, userName, logger) + if err != nil { + logger.Errorf("GetCollaborationNew error, err msg:%s", err) + return err + } + if updateResp.Update == nil && updateResp.UpdateInstance == nil && updateResp.New == nil && updateResp.Delete == nil { + return nil + } + err = syncInstance(updateResp, projectName, identityType, userName, uid, logger) + if err != nil { + logger.Errorf("syncInstance error, err msg:%s", err) + return err + } + err = syncResource(products, updateResp, projectName, identityType, userName, requestID, logger) + if err != nil { + logger.Errorf("syncResource error, err msg:%s", err) + return err + } + err = syncPolicy(updateResp, projectName, identityType, userName, uid, logger) + if err != nil { + logger.Errorf("syncPolicy error, err msg:%s", err) + return err + } + err = syncLabel(updateResp, projectName, identityType, userName, logger) + if err != nil { + logger.Errorf("syncLabel error, err msg:%s", err) + return err + } + return nil +} + +func getCollaborationDelete(updateResp *GetCollaborationUpdateResp) *GetCollaborationDeleteResp { + productSet := sets.String{} + workflowSet := sets.String{} + for _, item := range updateResp.Delete { + for _, product := range item.Products { + if product.CollaborationType == config.CollaborationNew { + productSet.Insert(product.Name) + } + } + for _, workflow := range item.Workflows { + if workflow.CollaborationType == config.CollaborationNew { + workflowSet.Insert(workflow.Name) + } + } + } + for _, item := range updateResp.Update { + for _, deleteWorkflow := range item.DeleteSpec.Workflows { + if deleteWorkflow.CollaborationType == config.CollaborationNew { + workflowSet.Insert(deleteWorkflow.Name) + } + } + for _, deleteProduct := range item.DeleteSpec.Products { + if deleteProduct.CollaborationType == config.CollaborationNew { + productSet.Insert(deleteProduct.Name) + } + } + for _, workflow := range item.UpdateSpec.Workflows { + if workflow.Old.CollaborationType == config.CollaborationNew && + workflow.New.CollaborationType == config.CollaborationShare { + workflowSet.Insert(workflow.Old.Name) + } + } + for _, product := range item.UpdateSpec.Products { + if product.Old.CollaborationType == config.CollaborationNew && + product.New.CollaborationType == config.CollaborationShare { + productSet.Insert(product.Old.Name) + } + } + } + return &GetCollaborationDeleteResp{ + Workflows: workflowSet.List(), + Products: productSet.List(), + } +} + +func getCollaborationNew(updateResp *GetCollaborationUpdateResp, projectName, identityType, userName string, + logger *zap.SugaredLogger) (*GetCollaborationNewResp, error) { + var newWorkflow []*Workflow + var newProduct []*Product + newProductName := sets.String{} + for _, mode := range updateResp.New { + for _, workflow := range mode.Workflows { + name := workflow.Name + if workflow.CollaborationType == config.CollaborationNew { + name = buildName(workflow.Name, mode.Name, identityType, userName) + } + newWorkflow = append(newWorkflow, &Workflow{ + CollaborationType: workflow.CollaborationType, + BaseName: workflow.Name, + CollaborationMode: mode.Name, + Name: name, + }) + } + for _, product := range mode.Products { + name := product.Name + if product.CollaborationType == config.CollaborationNew { + name = buildName(product.Name, mode.Name, identityType, userName) + } + newProduct = append(newProduct, &Product{ + CollaborationType: product.CollaborationType, + BaseName: product.Name, + CollaborationMode: mode.Name, + Name: name, + DeployType: mode.DeployType, + }) + newProductName.Insert(product.Name) + } + } + for _, item := range updateResp.Update { + for _, workflow := range item.NewSpec.Workflows { + name := workflow.Name + if workflow.CollaborationType == config.CollaborationNew { + name = buildName(workflow.Name, item.CollaborationMode, identityType, userName) + } + newWorkflow = append(newWorkflow, &Workflow{ + CollaborationType: workflow.CollaborationType, + BaseName: workflow.Name, + CollaborationMode: item.CollaborationMode, + Name: name, + }) + } + for _, product := range item.NewSpec.Products { + name := product.Name + if product.CollaborationType == config.CollaborationNew { + name = buildName(product.Name, item.CollaborationMode, identityType, userName) + } + newProduct = append(newProduct, &Product{ + CollaborationType: product.CollaborationType, + BaseName: product.Name, + CollaborationMode: item.CollaborationMode, + Name: name, + DeployType: item.DeployType, + }) + newProductName.Insert(product.Name) + } + for _, workflow := range item.UpdateSpec.Workflows { + if workflow.Old.CollaborationType == config.CollaborationShare && workflow.New.CollaborationType == config.CollaborationNew { + newWorkflow = append(newWorkflow, &Workflow{ + CollaborationType: workflow.New.CollaborationType, + BaseName: workflow.Old.BaseName, + CollaborationMode: item.CollaborationMode, + Name: buildName(workflow.Old.BaseName, item.CollaborationMode, identityType, userName), + }) + } + } + for _, product := range item.UpdateSpec.Products { + if product.Old.CollaborationType == config.CollaborationShare && product.New.CollaborationType == config.CollaborationNew { + newProduct = append(newProduct, &Product{ + CollaborationType: product.New.CollaborationType, + BaseName: product.Old.BaseName, + CollaborationMode: item.CollaborationMode, + DeployType: item.DeployType, + Name: buildName(product.Old.BaseName, item.CollaborationMode, identityType, userName), + }) + newProductName.Insert(product.Old.BaseName) + } + } + } + if len(newProduct) > 0 && newProduct[0].DeployType == setting.K8SDeployType { + renderSets, err := getRenderSet(projectName, newProductName.List()) + if err != nil { + logger.Errorf("getRenderSet error:%s", err) + return nil, err + } + if renderSets != nil { + productRenderSetMap := make(map[string]models2.RenderSet) + for _, set := range renderSets { + productRenderSetMap[set.EnvName] = set + } + for _, product := range newProduct { + set, ok := productRenderSetMap[product.BaseName] + if !ok { + logger.Warnf("product:%s renderSet not exist", product.BaseName) + continue + } + + product.Vars = set.KVs + product.DefaultValues = set.DefaultValues + } + } + } + if len(newProduct) > 0 && newProduct[0].DeployType == setting.HelmDeployType { + envChartsMap := getHelmRenderSet(projectName, newProductName.List(), logger) + for _, product := range newProduct { + chart, ok := envChartsMap[product.BaseName] + if !ok { + logger.Errorf("product:%s not exist", product.BaseName) + return nil, fmt.Errorf("product:%s not exist", product.BaseName) + } + + product.ChartValues = chart + } + } + var workNames []string + for _, workflow := range newWorkflow { + workNames = append(workNames, workflow.Name) + } + workflows, err := commonrepo.NewWorkflowColl().List(&commonrepo.ListWorkflowOption{ + Projects: []string{projectName}, + Names: workNames, + }) + if err != nil { + logger.Errorf("GetCollaborationNew list workflows:%v error:%s", workNames, err) + return nil, err + } + workflowDescMap := make(map[string]string) + for _, workflow := range workflows { + workflowDescMap[workflow.Name] = workflow.Description + } + for _, workflow := range newWorkflow { + if desc, ok := workflowDescMap[workflow.Name]; ok { + workflow.Description = desc + } + } + ifSync := false + if len(newWorkflow) == 0 && len(newProductName) == 0 && (len(updateResp.Update) != 0 || len(updateResp.Delete) != 0) { + ifSync = true + } + return &GetCollaborationNewResp{ + //Code 10000 means it is a filtered success result + Code: 10000, + Workflow: newWorkflow, + Product: newProduct, + IfSync: ifSync, + }, nil +} + +type DeleteCIResourcesRequest struct { + CollaborationInstances []models.CollaborationInstance `json:"collaboration_instances"` +} + +func CleanCIResources(userName, requestID string, logger *zap.SugaredLogger) error { + cis, err := mongodb.NewCollaborationInstanceColl().List(&mongodb.CollaborationInstanceFindOptions{}) + if err != nil { + return err + } + var fileterdInstances []*models.CollaborationInstance + for _, ci := range cis { + + if ci.RecycleDay != 0 && ((time.Now().Unix()-ci.LastVisitTime)/60 > ci.RecycleDay*24*60) { + fileterdInstances = append(fileterdInstances, ci) + } + } + return DeleteCIResources(userName, requestID, fileterdInstances, logger) +} + +func DeleteCIResources(userName, requestID string, cis []*models.CollaborationInstance, logger *zap.SugaredLogger) error { + var names string + var policyNames []string + if len(cis) == 0 { + return nil + } + var findOpts []mongodb.CollaborationInstanceFindOptions + + for _, ci := range cis { + findOpts = append(findOpts, mongodb.CollaborationInstanceFindOptions{ + ProjectName: ci.ProjectName, + Name: ci.CollaborationName, + UserUID: []string{ci.UserUID}, + }) + policyNames = append(policyNames, ci.PolicyName) + } + + err := mongodb.NewCollaborationInstanceColl().BulkDelete(mongodb.CollaborationInstanceListOptions{ + FindOpts: findOpts, + }) + if err != nil { + logger.Errorf("BulkDelete CollaborationInstance error:%s", err) + return err + } + names = cis[0].PolicyName + for i := 1; i < len(cis); i++ { + names = names + "," + cis[i].PolicyName + } + res, err := policy.NewDefault().GetPolicies(names) + if err != nil { + return err + } + + var labels []mongodb2.Label + labelSet := sets.String{} + for _, re := range res { + for _, rule := range re.Rules { + for _, attribute := range rule.MatchAttributes { + if attribute.Key != "placeholder" && attribute.Key != "production" && + !labelSet.Has(attribute.Key+"-"+attribute.Value) { + labels = append(labels, mongodb2.Label{ + Key: attribute.Key, + Value: attribute.Value, + }) + labelSet.Insert(attribute.Key + "-" + attribute.Value) + } + } + } + } + + labelRes, err := service.ListLabels(&service.ListLabelsArgs{ + Labels: labels, + }) + if err != nil { + return err + } + var labelIds []string + for _, l := range labelRes.Labels { + labelIds = append(labelIds, l.ID.Hex()) + } + + err = service.DeleteLabels(labelIds, true, "system", logger) + if err != nil { + return err + } + for _, ci := range cis { + for _, workflow := range ci.Workflows { + if workflow.CollaborationType == config.CollaborationNew { + err = commonservice.DeleteWorkflow(workflow.Name, requestID, false, logger) + if err != nil { + return err + } + } + } + for _, product := range ci.Products { + if product.CollaborationType == config.CollaborationNew { + err = service2.DeleteProduct(userName, product.Name, ci.ProjectName, requestID, true, logger) + if err != nil { + return err + } + } + } + } + err = policy.NewDefault().DeletePolicies("", policy.DeletePoliciesArgs{ + Names: policyNames, + }) + if err != nil { + logger.Errorf("BulkDelete policy error:%s", err) + return err + } + + return nil +} + +func GetCollaborationNew(projectName, uid, identityType, userName string, logger *zap.SugaredLogger) (*GetCollaborationNewResp, error) { + updateResp, err := GetCollaborationUpdate(projectName, uid, identityType, userName, logger) + if err != nil { + logger.Errorf("GetCollaborationNew error, err msg:%s", err) + return nil, err + } + if updateResp == nil || (updateResp.Update == nil && updateResp.New == nil && updateResp.UpdateInstance == nil && updateResp.Delete == nil) { + return nil, nil + } + return getCollaborationNew(updateResp, projectName, identityType, userName, logger) +} + +func getRenderSet(projectName string, envs []string) ([]models2.RenderSet, error) { + products, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{ + InProjects: []string{projectName}, + InEnvs: envs, + }) + if err != nil { + return nil, err + } + var findOpts []commonrepo.RenderSetFindOption + for _, product := range products { + var revision int64 + for _, productService := range product.GetServiceMap() { + revision = productService.Render.Revision + } + findOpts = append(findOpts, commonrepo.RenderSetFindOption{ + Revision: revision, + ProductTmpl: projectName, + EnvName: product.EnvName, + Name: product.Namespace, + }) + } + renderSets, err := commonrepo.NewRenderSetColl().ListByFindOpts(&commonrepo.RenderSetListOption{ + ProductTmpl: projectName, + FindOpts: findOpts, + }) + if err != nil { + return nil, err + } + return renderSets, nil +} + +func getHelmRenderSet(projectName string, envs []string, logger *zap.SugaredLogger) map[string][]*commonservice.RenderChartArg { + envChartsMap := make(map[string][]*commonservice.RenderChartArg) + for _, env := range envs { + renderChartArgs, err := commonservice.GetRenderCharts(projectName, env, "", logger) + if err != nil { + logger.Errorf("GetRenderCharts error:%s", err) + continue + } + envChartsMap[env] = renderChartArgs + } + + return envChartsMap +} diff --git a/pkg/tool/ilyshin/branch.go b/pkg/microservice/aslan/core/collaboration/service/collaboration_mode.go similarity index 34% rename from pkg/tool/ilyshin/branch.go rename to pkg/microservice/aslan/core/collaboration/service/collaboration_mode.go index aa168d3eee44ded10e3e6e3a1418cbc60c9e55e4..0771d79c2bcb37075b881766856df0a78b7f41de 100644 --- a/pkg/tool/ilyshin/branch.go +++ b/pkg/microservice/aslan/core/collaboration/service/collaboration_mode.go @@ -14,44 +14,38 @@ See the License for the specific language governing permissions and limitations under the License. */ -package ilyshin +package service import ( - "fmt" - "net/url" - "go.uber.org/zap" - "github.com/koderover/zadig/pkg/tool/httpclient" + "github.com/koderover/zadig/pkg/microservice/aslan/core/collaboration/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/collaboration/repository/mongodb" ) -type Branch struct { - Name string `json:"name"` - Protected bool `json:"protected"` - Merged bool `json:"merged"` - Default bool `json:"default"` - CanPush bool `json:"can_push"` - DevelopersCanPush bool `json:"developers_can_push"` - DevelopersCanMerge bool `json:"developers_can_merge"` - Commit *Commit `json:"commit"` -} - -func (c *Client) ListBranches(owner, repo string, log *zap.SugaredLogger) ([]*Branch, error) { - url := fmt.Sprintf("/api/v4/projects/%s/repository/branches", generateProjectName(owner, repo)) - qs := map[string]string{ - "per_page": "100", +func CreateCollaborationMode(userName string, collaborationMode *models.CollaborationMode, logger *zap.SugaredLogger) error { + err := mongodb.NewCollaborationModeColl().Create(userName, collaborationMode) + if err != nil { + logger.Errorf("CreateCollaborationMode error, err msg:%s", err) + return err } + return nil +} - var err error - var branches []*Branch - if _, err = c.Get(url, httpclient.SetQueryParams(qs), httpclient.SetResult(&branches)); err != nil { - log.Errorf("Failed to list project branches, error: %s", err) - return branches, err +func UpdateCollaborationMode(userName string, collaborationMode *models.CollaborationMode, logger *zap.SugaredLogger) error { + err := mongodb.NewCollaborationModeColl().Update(userName, collaborationMode) + if err != nil { + logger.Errorf("UpdateCollaborationMode error, err msg:%s", err) + return err } - - return branches, nil + return nil } -func generateProjectName(owner, repo string) string { - return url.PathEscape(fmt.Sprintf("%s/%s", owner, repo)) +func DeleteCollaborationMode(username, projectName, name string, logger *zap.SugaredLogger) error { + err := mongodb.NewCollaborationModeColl().Delete(username, projectName, name) + if err != nil { + logger.Errorf("UpdateCollaborationMode error, err msg:%s", err) + return err + } + return mongodb.NewCollaborationInstanceColl().ResetRevision(name, projectName) } diff --git a/pkg/microservice/aslan/core/common/handler/callback.go b/pkg/microservice/aslan/core/common/handler/callback.go new file mode 100644 index 0000000000000000000000000000000000000000..212e347c3549a8154f1b9f063330a1f97b1eb47b --- /dev/null +++ b/pkg/microservice/aslan/core/common/handler/callback.go @@ -0,0 +1,44 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "fmt" + + "github.com/gin-gonic/gin" + + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + "github.com/koderover/zadig/pkg/tool/log" +) + +func HandleCallback(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + req := &commonmodels.CallbackRequest{} + err := c.ShouldBindJSON(req) + if err != nil { + log.Errorf("failed to validate callback request, the error is: %s", err) + ctx.Err = fmt.Errorf("failed to validate callback request, the error is: %s", err) + return + } + + ctx.Err = commonservice.HandleCallback(req) + +} diff --git a/pkg/microservice/aslan/core/common/repository/models/basic_images.go b/pkg/microservice/aslan/core/common/repository/models/basic_images.go index d250ddb463f3facf49de763a2d0bdb84e268a944..eaf81186cacc78f31daa2c0c9de4a0649af50a34 100644 --- a/pkg/microservice/aslan/core/common/repository/models/basic_images.go +++ b/pkg/microservice/aslan/core/common/repository/models/basic_images.go @@ -34,6 +34,9 @@ type BasicImage struct { CreateTime int64 `bson:"create_time" json:"create_time"` UpdateTime int64 `bson:"update_time" json:"update_time"` UpdateBy string `bson:"update_by" json:"update_by"` + // New field since 1.12 + // ImageType is the type of the image, currently only one type is supported called sonar + ImageType string `bson:"image_type" json:"image_type"` } func (BasicImage) TableName() string { diff --git a/pkg/microservice/aslan/core/common/repository/models/build.go b/pkg/microservice/aslan/core/common/repository/models/build.go index e808b529ac608f589ce0a556d2588ab75d201405..20b4926575eceaeffa24ad384ae49a6129fa1ec1 100644 --- a/pkg/microservice/aslan/core/common/repository/models/build.go +++ b/pkg/microservice/aslan/core/common/repository/models/build.go @@ -24,34 +24,47 @@ import ( ) type Build struct { - ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` - Name string `bson:"name" json:"name"` - Team string `bson:"team,omitempty" json:"team,omitempty"` - Source string `bson:"source,omitempty" json:"source,omitempty"` - Timeout int `bson:"timeout" json:"timeout"` + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + Name string `bson:"name" json:"name"` + Team string `bson:"team,omitempty" json:"team,omitempty"` + Source string `bson:"source,omitempty" json:"source,omitempty"` + TemplateID string `bson:"template_id" json:"template_id"` + Timeout int `bson:"timeout" json:"timeout"` // 在任一编译配置模板中只能出现一次 // 对于k8s部署是传入容器名称 // 对于物理机部署是服务名称 - Targets []*ServiceModuleTarget `bson:"targets" json:"targets"` - Description string `bson:"desc,omitempty" json:"desc"` - UpdateTime int64 `bson:"update_time" json:"update_time"` - UpdateBy string `bson:"update_by" json:"update_by"` - Repos []*types.Repository `bson:"repos" json:"repos"` - PreBuild *PreBuild `bson:"pre_build" json:"pre_build"` - JenkinsBuild *JenkinsBuild `bson:"jenkins_build,omitempty" json:"jenkins_build,omitempty"` - Scripts string `bson:"scripts" json:"scripts"` - PostBuild *PostBuild `bson:"post_build,omitempty" json:"post_build"` - Caches []string `bson:"caches" json:"caches"` - ProductName string `bson:"product_name" json:"product_name"` - SSHs []string `bson:"sshs,omitempty" json:"sshs,omitempty"` - PMDeployScripts string `bson:"pm_deploy_scripts" json:"pm_deploy_scripts"` + Targets []*ServiceModuleTarget `bson:"targets" json:"targets"` + TargetRepos []*TargetRepo `bson:"-" json:"target_repos"` + Description string `bson:"desc,omitempty" json:"desc"` + UpdateTime int64 `bson:"update_time" json:"update_time"` + UpdateBy string `bson:"update_by" json:"update_by"` + Repos []*types.Repository `bson:"repos" json:"repos"` + PreBuild *PreBuild `bson:"pre_build" json:"pre_build"` + JenkinsBuild *JenkinsBuild `bson:"jenkins_build,omitempty" json:"jenkins_build,omitempty"` + Scripts string `bson:"scripts" json:"scripts"` + PostBuild *PostBuild `bson:"post_build,omitempty" json:"post_build"` + + // TODO: Deprecated. + Caches []string `bson:"caches" json:"caches"` + ProductName string `bson:"product_name" json:"product_name"` + SSHs []string `bson:"sshs" json:"sshs"` + PMDeployScripts string `bson:"pm_deploy_scripts" json:"pm_deploy_scripts"` + + // New since V1.10.0. + CacheEnable bool `bson:"cache_enable" json:"cache_enable"` + CacheDirType types.CacheDirType `bson:"cache_dir_type" json:"cache_dir_type"` + CacheUserDir string `bson:"cache_user_dir" json:"cache_user_dir"` + // New since V1.10.0. Only to tell the webpage should the advanced settings be displayed + AdvancedSettingsModified bool `bson:"advanced_setting_modified" json:"advanced_setting_modified"` } // PreBuild prepares an environment for a job type PreBuild struct { + // TODO: Deprecated. CleanWorkspace bool `bson:"clean_workspace" json:"clean_workspace"` // ResReq defines job requested resources - ResReq setting.Request `bson:"res_req" json:"res_req"` + ResReq setting.Request `bson:"res_req" json:"res_req"` + ResReqSpec setting.RequestSpec `bson:"res_req_spec" json:"res_req_spec"` // BuildOS defines job image OS, it supports 12.04, 14.04, 16.04 BuildOS string `bson:"build_os" json:"build_os"` ImageFrom string `bson:"image_from" json:"image_from"` @@ -65,7 +78,11 @@ type PreBuild struct { // Parameters Parameters []*Parameter `bson:"parameters,omitempty" json:"parameters"` // UploadPkg uploads package to s3 - UploadPkg bool `bson:"upload_pkg" json:"upload_pkg"` + UploadPkg bool `bson:"upload_pkg" json:"upload_pkg"` + ClusterID string `bson:"cluster_id" json:"cluster_id"` + + // TODO: Deprecated. + Namespace string `bson:"namespace" json:"namespace"` } type BuildObj struct { @@ -80,15 +97,22 @@ type BuildObj struct { } type PostBuild struct { - DockerBuild *DockerBuild `bson:"docker_build,omitempty" json:"docker_build"` - FileArchive *FileArchive `bson:"file_archive,omitempty" json:"file_archive,omitempty"` - Scripts string `bson:"scripts" json:"scripts"` + DockerBuild *DockerBuild `bson:"docker_build,omitempty" json:"docker_build"` + ObjectStorageUpload *ObjectStorageUpload `bson:"object_storage_upload" json:"object_storage_upload"` + FileArchive *FileArchive `bson:"file_archive,omitempty" json:"file_archive,omitempty"` + Scripts string `bson:"scripts" json:"scripts"` } type FileArchive struct { FileLocation string `bson:"file_location" json:"file_location"` } +type ObjectStorageUpload struct { + Enabled bool `bson:"enabled" json:"enabled"` + ObjectStorageID string `bson:"object_storage_id" json:"object_storage_id"` + UploadDetail []*types.ObjectStoragePathDetail `bson:"upload_detail" json:"upload_detail"` +} + type DockerBuild struct { // WorkDir docker run path WorkDir string `bson:"work_dir" json:"work_dir"` @@ -96,16 +120,18 @@ type DockerBuild struct { DockerFile string `bson:"docker_file" json:"docker_file"` // BuildArgs docker build args BuildArgs string `bson:"build_args,omitempty" json:"build_args"` + // Source whether dockerfile comes from template or existing file + Source string `bson:"source" json:"source"` + // TemplateId is the id of the template dockerfile + TemplateID string `bson:"template_id" json:"template_id"` + // TemplateName is the name of the template dockerfile + TemplateName string `bson:"template_name" json:"template_name"` } type JenkinsBuild struct { - JobName string `bson:"job_name" json:"job_name"` - JenkinsBuildParam []*JenkinsBuildParam `bson:"jenkins_build_param" json:"jenkins_build_params"` -} - -type JenkinsBuildParam struct { - Name string `json:"name"` - Value interface{} `json:"value"` + JenkinsID string `bson:"jenkins_id" json:"jenkins_id"` + JobName string `bson:"job_name" json:"job_name"` + JenkinsBuildParam []*types.JenkinsBuildParam `bson:"jenkins_build_param" json:"jenkins_build_params"` } type Parameter struct { @@ -121,15 +147,32 @@ type ParamVal struct { } type ServiceModuleTarget struct { - ProductName string `bson:"product_name" json:"product_name"` - ServiceName string `bson:"service_name" json:"service_name"` - ServiceModule string `bson:"service_module" json:"service_module"` + ProductName string `bson:"product_name" json:"product_name"` + ServiceName string `bson:"service_name" json:"service_name"` + ServiceModule string `bson:"service_module" json:"service_module"` + BuildName string `bson:"build_name" json:"build_name"` + Repos []*types.Repository `bson:"repos,omitempty" json:"repos,omitempty"` + Envs []*KeyVal `bson:"envs,omitempty" json:"envs"` +} + +type ServiceModuleTargetBase struct { + ProductName string `json:"product_name"` + ServiceName string `json:"service_name"` + ServiceModule string `json:"service_module"` +} + +type TargetRepo struct { + Service *ServiceModuleTargetBase `json:"service"` + Repos []*types.Repository `json:"repos"` + Envs []*KeyVal `json:"envs"` } type KeyVal struct { - Key string `bson:"key" json:"key"` - Value string `bson:"value" json:"value"` - IsCredential bool `bson:"is_credential" json:"is_credential"` + Key string `bson:"key" json:"key" yaml:"key"` + Value string `bson:"value" json:"value" yaml:"value"` + Type ParameterSettingType `bson:"type,omitempty" json:"type,omitempty" yaml:"type"` + ChoiceOption []string `bson:"choice_option,omitempty" json:"choice_option,omitempty" yaml:"choice_option,omitempty"` + IsCredential bool `bson:"is_credential" json:"is_credential" yaml:"is_credential"` } type Item struct { @@ -137,6 +180,25 @@ type Item struct { Version string `bson:"version" json:"version"` } +func (build *Build) SafeRepos() []*types.Repository { + if len(build.Repos) == 0 { + return []*types.Repository{} + } + return build.Repos +} + +func (build *Build) SafeReposDeepCopy() []*types.Repository { + if len(build.Repos) == 0 { + return []*types.Repository{} + } + resp := make([]*types.Repository, 0) + for _, repo := range build.Repos { + tmpRepo := *repo + resp = append(resp, &tmpRepo) + } + return resp +} + func (Build) TableName() string { return "module_build" } diff --git a/pkg/microservice/aslan/core/common/repository/models/build_template.go b/pkg/microservice/aslan/core/common/repository/models/build_template.go new file mode 100644 index 0000000000000000000000000000000000000000..8dc3f1e6660c9271f7d75040fcca60b4a878182d --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/models/build_template.go @@ -0,0 +1,47 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" + + "github.com/koderover/zadig/pkg/types" +) + +type BuildTemplate struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + Name string `bson:"name" json:"name"` + Team string `bson:"team,omitempty" json:"team,omitempty"` + Source string `bson:"source,omitempty" json:"source,omitempty"` + Timeout int `bson:"timeout" json:"timeout"` + UpdateTime int64 `bson:"update_time" json:"update_time"` + UpdateBy string `bson:"update_by" json:"update_by"` + PreBuild *PreBuild `bson:"pre_build" json:"pre_build"` + JenkinsBuild *JenkinsBuild `bson:"jenkins_build,omitempty" json:"jenkins_build,omitempty"` + Scripts string `bson:"scripts" json:"scripts"` + PostBuild *PostBuild `bson:"post_build,omitempty" json:"post_build"` + SSHs []string `bson:"sshs" json:"sshs"` + PMDeployScripts string `bson:"pm_deploy_scripts" json:"pm_deploy_scripts"` + CacheEnable bool `bson:"cache_enable" json:"cache_enable"` + CacheDirType types.CacheDirType `bson:"cache_dir_type" json:"cache_dir_type"` + CacheUserDir string `bson:"cache_user_dir" json:"cache_user_dir"` + AdvancedSettingsModified bool `bson:"advanced_setting_modified" json:"advanced_setting_modified"` +} + +func (BuildTemplate) TableName() string { + return "build_template" +} diff --git a/pkg/microservice/aslan/internal/cache/cache.go b/pkg/microservice/aslan/core/common/repository/models/callback_request.go similarity index 40% rename from pkg/microservice/aslan/internal/cache/cache.go rename to pkg/microservice/aslan/core/common/repository/models/callback_request.go index b62663558d549d013634c0dfcdc39b2f45e3811e..28bc6f0ff726c9e08d33315a7612976aa5aa1e78 100644 --- a/pkg/microservice/aslan/internal/cache/cache.go +++ b/pkg/microservice/aslan/core/common/repository/models/callback_request.go @@ -14,38 +14,22 @@ See the License for the specific language governing permissions and limitations under the License. */ -package cache - -import ( - "sync" - - "github.com/coocood/freecache" -) - -const defaultSize = 1024 * 1024 * 10 // 10 Mi - -var once sync.Once -var cache *freecache.Cache - -// Set sets a key, value and expiration for a cache entry and stores it in the cache. -// If the key is larger than 65535 or value is larger than 1/1024 of the cache size, -// the entry will not be written to the cache. expireSeconds <= 0 means no expire, -// but it can be evicted when cache is full. -func Set(key, value []byte, expireSeconds int) error { - once.Do(func() { - cache = initCache(0) - }) - return cache.Set(key, value, expireSeconds) -} - -// Get returns the value or not found error. -func Get(key []byte) ([]byte, error) { - once.Do(func() { - cache = initCache(defaultSize) - }) - return cache.Get(key) +package models + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type CallbackRequest struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + Type string `bson:"type" json:"type"` + // this is called pipeline name because it is called so all around this system + PipelineName string `bson:"task_name" json:"task_name"` + ProjectName string `bson:"project_name" json:"project_name"` + TaskID int64 `bson:"task_id" json:"task_id"` + Status string `bson:"status" json:"status"` + StatusMessage string `bson:"status_message" json:"status_message"` + Payload map[string]interface{} `bson:"payload" json:"payload"` } -func initCache(size int) *freecache.Cache { - return freecache.NewCache(size) +func (CallbackRequest) TableName() string { + return "callback_request" } diff --git a/pkg/microservice/aslan/core/common/repository/models/chart.go b/pkg/microservice/aslan/core/common/repository/models/chart.go new file mode 100644 index 0000000000000000000000000000000000000000..0f9434fa8d3729f2895b65066935358e6784165f --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/models/chart.go @@ -0,0 +1,62 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type Chart struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + Name string `bson:"name" json:"name"` + Source string `bson:"source" json:"source"` + Owner string `bson:"owner" json:"owner"` + Namespace string `bson:"namespace" json:"namespace"` + Repo string `bson:"repo" json:"repo"` + Path string `bson:"path" json:"path"` + Branch string `bson:"branch" json:"branch"` + CodeHostID int `bson:"codehost_id" json:"codeHostID"` + Revision int64 `bson:"revision" json:"revision"` + ChartVariables []*ChartVariable `bson:"variables" json:"variables"` + Sha1 string `bson:"sha1" json:"sha1"` +} + +type ChartVariable struct { + Key string `json:"key"` + Value string `json:"value,omitempty"` + Description string `json:"description,omitempty"` +} + +func (chart *Chart) GetNamespace() string { + if len(chart.Namespace) > 0 { + return chart.Namespace + } + return chart.Owner +} + +func (chart *Chart) GetVariableMap() map[string]*ChartVariable { + if len(chart.ChartVariables) == 0 { + return nil + } + ret := make(map[string]*ChartVariable) + for _, v := range chart.ChartVariables { + ret[v.Key] = v + } + return ret +} + +func (Chart) TableName() string { + return "chart_template" +} diff --git a/pkg/microservice/aslan/core/common/repository/models/delivery_distribute.go b/pkg/microservice/aslan/core/common/repository/models/delivery_distribute.go index db24294d6654dad48afde50f71c224c24ec35cb1..bbe82db54e58b5b054f326d54a5c52e6ca4e4aec 100644 --- a/pkg/microservice/aslan/core/common/repository/models/delivery_distribute.go +++ b/pkg/microservice/aslan/core/common/repository/models/delivery_distribute.go @@ -23,16 +23,23 @@ import ( ) type DeliveryDistribute struct { - ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` ReleaseID primitive.ObjectID `bson:"release_id" json:"releaseId"` - ServiceName string `bson:"service_name" json:"serviceName"` + ServiceName string `bson:"service_name" json:"serviceName,omitempty"` DistributeType config.DistributeType `bson:"distribute_type" json:"distributeType"` RegistryName string `bson:"registry_name" json:"registryName"` - Namespace string `bson:"namespace" json:"namespace"` - PackageFile string `bson:"package_file" json:"packageFile"` - RemoteFileKey string `bson:"remote_file_key" json:"remoteFileKey"` - DestStorageURL string `bson:"dest_storage_url" json:"destStorageUrl"` - SrcStorageURL string `bson:"src_storage_url" json:"srcStorageUrl"` + ChartVersion string `bson:"chart_version" json:"chartVersion,omitempty"` + ChartName string `bson:"chart_name" json:"chartName,omitempty"` + ChartRepoName string `bson:"chart_repo_name" json:"chartRepoName,omitempty"` + SubDistributes []*DeliveryDistribute `bson:"-" json:"subDistributes,omitempty"` + Namespace string `bson:"namespace" json:"namespace,omitempty"` + PackageFile string `bson:"package_file" json:"packageFile,omitempty"` + RemoteFileKey string `bson:"remote_file_key" json:"remoteFileKey,omitempty"` + DestStorageURL string `bson:"dest_storage_url" json:"destStorageUrl,omitempty"` + S3StorageID string `bson:"s3_storage_id" json:"s3StorageID"` + StorageURL string `bson:"-" json:"storageUrl"` + StorageBucket string `bson:"-" json:"storageBucket"` + SrcStorageURL string `bson:"src_storage_url" json:"srcStorageUrl,omitempty"` StartTime int64 `bson:"start_time,omitempty" json:"start_time,omitempty"` EndTime int64 `bson:"end_time,omitempty" json:"end_time,omitempty"` CreatedAt int64 `bson:"created_at" json:"created_at"` diff --git a/pkg/microservice/aslan/core/common/repository/models/delivery_version.go b/pkg/microservice/aslan/core/common/repository/models/delivery_version.go index 785bc63842f9d71df6a76eee3de96706fdb96c5d..ed851379dcb5b451b4e5379add4777f64ffa5aa9 100644 --- a/pkg/microservice/aslan/core/common/repository/models/delivery_version.go +++ b/pkg/microservice/aslan/core/common/repository/models/delivery_version.go @@ -20,19 +20,30 @@ import ( "go.mongodb.org/mongo-driver/bson/primitive" ) +type DeliveryVersionProgress struct { + SuccessChartCount int `json:"successChartCount"` + TotalChartCount int `json:"totalChartCount"` + PackageUploadStatus string `json:"packageStatus"` + Error string `json:"error"` +} + type DeliveryVersion struct { - ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` - OrgID int `bson:"org_id" json:"orgId"` - Version string `bson:"version" json:"version"` - ProductName string `bson:"product_name" json:"productName"` - WorkflowName string `bson:"workflow_name" json:"workflowName"` - TaskID int `bson:"task_id" json:"taskId"` - Desc string `bson:"desc" json:"desc"` - Labels []string `bson:"labels" json:"labels"` - ProductEnvInfo *Product `bson:"product_env_info" json:"productEnvInfo"` - CreatedBy string `bson:"created_by" json:"createdBy"` - CreatedAt int64 `bson:"created_at" json:"created_at"` - DeletedAt int64 `bson:"deleted_at" json:"deleted_at"` + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + Version string `bson:"version" json:"version"` + ProductName string `bson:"product_name" json:"productName"` + WorkflowName string `bson:"workflow_name" json:"workflowName"` + Type string `bson:"type" json:"type"` + TaskID int `bson:"task_id" json:"taskId"` + Desc string `bson:"desc" json:"desc"` + Labels []string `bson:"labels" json:"labels"` + ProductEnvInfo *Product `bson:"product_env_info" json:"productEnvInfo"` + Status string `bson:"status" json:"status"` + Error string `bson:"error" json:"-"` + Progress *DeliveryVersionProgress `bson:"-" json:"progress"` + CreateArgument interface{} `bson:"createArgument" json:"-"` + CreatedBy string `bson:"created_by" json:"createdBy"` + CreatedAt int64 `bson:"created_at" json:"created_at"` + DeletedAt int64 `bson:"deleted_at" json:"deleted_at"` } func (DeliveryVersion) TableName() string { diff --git a/pkg/microservice/aslan/core/common/repository/models/dind_clean.go b/pkg/microservice/aslan/core/common/repository/models/dind_clean.go index 17c5b9b72bc8167c3c946caaeff2ca8a884f7cec..5ac3935f50f8b45fb65504db9b946a45344485a2 100644 --- a/pkg/microservice/aslan/core/common/repository/models/dind_clean.go +++ b/pkg/microservice/aslan/core/common/repository/models/dind_clean.go @@ -26,6 +26,8 @@ type DindClean struct { Status string `bson:"status" json:"status"` UpdateTime int64 `bson:"update_time" json:"update_time"` DindCleanInfos []*DindCleanInfo `bson:"dind_clean_infos" json:"dind_clean_infos"` + Cron string `bson:"cron" json:"cron"` + CronEnabled bool `bson:"cron_enabled" json:"cron_enabled"` } type DindCleanInfo struct { diff --git a/pkg/microservice/aslan/core/common/repository/models/dockerfile.go b/pkg/microservice/aslan/core/common/repository/models/dockerfile.go new file mode 100644 index 0000000000000000000000000000000000000000..97af86c54d74cb43f2dabdc726d394ac75dd2be1 --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/models/dockerfile.go @@ -0,0 +1,29 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type DockerfileTemplate struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + Name string `bson:"name" json:"name"` + Content string `bson:"content" json:"content"` +} + +func (DockerfileTemplate) TableName() string { + return "dockerfile_template" +} diff --git a/pkg/microservice/aslan/core/common/repository/models/env_resource.go b/pkg/microservice/aslan/core/common/repository/models/env_resource.go new file mode 100644 index 0000000000000000000000000000000000000000..bc71ce7e9844d6d1c1c8a3c65559d7e9a1cadcbf --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/models/env_resource.go @@ -0,0 +1,40 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type EnvResource struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + Type string `bson:"type" json:"type"` + ProductName string `bson:"product_name" json:"product_name"` + CreateTime int64 `bson:"create_time" json:"create_time"` + UpdateUserName string `bson:"update_user_name" json:"update_user_name"` + DeletedAt int64 `bson:"deleted_at" json:"deleted_at" ` + Namespace string `bson:"namespace,omitempty" json:"namespace,omitempty"` + EnvName string `bson:"env_name" json:"env_name"` + Name string `bson:"name" json:"name"` + YamlData string `bson:"yaml_data" json:"yaml_data"` + SourceDetail *CreateFromRepo `bson:"source_detail" json:"source_detail"` + AutoSync bool `bson:"auto_sync" json:"auto_sync"` +} + +func (EnvResource) TableName() string { + return "env_resource" +} diff --git a/pkg/microservice/aslan/core/common/repository/models/env_svc_depend.go b/pkg/microservice/aslan/core/common/repository/models/env_svc_depend.go new file mode 100644 index 0000000000000000000000000000000000000000..e95836d4fda33d1ba1b62a6655270bdd980107f2 --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/models/env_svc_depend.go @@ -0,0 +1,37 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type EnvSvcDepend struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + ProductName string `bson:"product_name" json:"product_name"` + Namespace string `bson:"namespace,omitempty" json:"namespace,omitempty"` + EnvName string `bson:"env_name" json:"env_name"` + ServiceName string `bson:"service_name" json:"service_name"` + ServiceModule string `bson:"service_module" json:"service_module"` + WorkloadType string `bson:"workload_type" json:"workload_type"` + Pvcs []string `bson:"pvcs" json:"pvcs"` + ConfigMaps []string `bson:"configmaps" json:"configmaps"` + Secrets []string `bson:"secrets" json:"secrets"` + CreateTime int64 `bson:"create_time" json:"create_time"` +} + +func (EnvSvcDepend) TableName() string { + return "env_svc_depend" +} diff --git a/pkg/microservice/aslan/core/common/repository/models/external_link.go b/pkg/microservice/aslan/core/common/repository/models/external_link.go new file mode 100644 index 0000000000000000000000000000000000000000..beb086a52ba561eab44f6c4c00a61652f599f0e3 --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/models/external_link.go @@ -0,0 +1,32 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type ExternalLink struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + Name string `bson:"name" json:"name"` + URL string `bson:"url" json:"url"` + CreateTime int64 `bson:"create_time" json:"create_time"` + UpdateTime int64 `bson:"update_time" json:"update_time"` + UpdateBy string `bson:"update_by" json:"update_by"` +} + +func (ExternalLink) TableName() string { + return "external_link" +} diff --git a/pkg/microservice/aslan/core/common/repository/models/external_system.go b/pkg/microservice/aslan/core/common/repository/models/external_system.go new file mode 100644 index 0000000000000000000000000000000000000000..08d7e69eec1a64323064efcc938ddd4f4c5b5410 --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/models/external_system.go @@ -0,0 +1,30 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type ExternalSystem struct { + ID primitive.ObjectID `bson:"_id,omitempty"` + Name string `bson:"name"` + Server string `bson:"server"` + APIToken string `bson:"api_token"` +} + +func (ExternalSystem) TableName() string { + return "external_system" +} diff --git a/pkg/microservice/aslan/core/common/repository/models/favorite_pipeline.go b/pkg/microservice/aslan/core/common/repository/models/favorite_pipeline.go index 5e1ab74ef06cb8b1c32802c05effec26fae8ba91..b0bca8b794cfd48041c0ef5e182d6d5040868fe4 100644 --- a/pkg/microservice/aslan/core/common/repository/models/favorite_pipeline.go +++ b/pkg/microservice/aslan/core/common/repository/models/favorite_pipeline.go @@ -20,7 +20,7 @@ import "go.mongodb.org/mongo-driver/bson/primitive" type Favorite struct { ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` - UserID int `bson:"user_id" json:"user_id"` + UserID string `bson:"user_id" json:"user_id"` ProductName string `bson:"product_name" json:"product_name"` Name string `bson:"name" json:"name"` Type string `bson:"type" json:"type"` diff --git a/pkg/microservice/aslan/core/common/repository/models/k8s_cluster.go b/pkg/microservice/aslan/core/common/repository/models/k8s_cluster.go index f3f60b219c1008d7ff232a2763535c767ba84269..5339b5c36e13a5857f83db9a8ae14702bd93d780 100644 --- a/pkg/microservice/aslan/core/common/repository/models/k8s_cluster.go +++ b/pkg/microservice/aslan/core/common/repository/models/k8s_cluster.go @@ -17,33 +17,41 @@ limitations under the License. package models import ( - "fmt" - "regexp" - "strings" - "go.mongodb.org/mongo-driver/bson/primitive" + corev1 "k8s.io/api/core/v1" - "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/types" ) -var namePattern = regexp.MustCompile(`^[0-9a-zA-Z_.-]{1,32}$`) - type K8SCluster struct { - ID primitive.ObjectID `json:"id,omitempty" bson:"_id,omitempty"` - Name string `json:"name" bson:"name"` - Tags []string `json:"tags" bson:"tags"` - Description string `json:"description" bson:"description"` - Namespace string `json:"namespace" bson:"namespace"` - Info *K8SClusterInfo `json:"info,omitempty" bson:"info,omitempty"` - Status config.K8SClusterStatus `json:"status" bson:"status"` - Error string `json:"error" bson:"error"` - Yaml string `json:"yaml" bson:"yaml"` - Production bool `json:"production" bson:"production"` - CreatedAt int64 `json:"createdAt" bson:"createdAt"` - CreatedBy string `json:"createdBy" bson:"createdBy"` - Disconnected bool `json:"-" bson:"disconnected"` - Token string `json:"token" bson:"-"` - Provider int8 `json:"provider" bson:"provider"` + ID primitive.ObjectID `json:"id,omitempty" bson:"_id,omitempty"` + Name string `json:"name" bson:"name"` + Tags []string `json:"tags" bson:"tags"` + Description string `json:"description" bson:"description"` + Namespace string `json:"namespace" bson:"namespace"` + Info *K8SClusterInfo `json:"info,omitempty" bson:"info,omitempty"` + AdvancedConfig *AdvancedConfig `json:"advanced_config,omitempty" bson:"advanced_config,omitempty"` + Status setting.K8SClusterStatus `json:"status" bson:"status"` + Error string `json:"error" bson:"error"` + Yaml string `json:"yaml" bson:"yaml"` + Production bool `json:"production" bson:"production"` + CreatedAt int64 `json:"createdAt" bson:"createdAt"` + CreatedBy string `json:"createdBy" bson:"createdBy"` + Disconnected bool `json:"-" bson:"disconnected"` + Token string `json:"token" bson:"-"` + Provider int8 `json:"provider" bson:"provider"` + Local bool `json:"local" bson:"local"` + Cache types.Cache `json:"cache" bson:"cache"` + LastConnectionTime int64 `json:"last_connection_time" bson:"last_connection_time"` + UpdateHubagentErrorMsg string `json:"update_hubagent_error_msg" bson:"update_hubagent_error_msg"` + DindCfg *DindCfg `json:"dind_cfg" bson:"dind_cfg"` +} + +type K8SClusterResp struct { + ID string `json:"id" bson:"id,omitempty"` + Name string `json:"name" bson:"name"` + AdvancedConfig *AdvancedConfig `json:"advanced_config,omitempty" bson:"advanced_config,omitempty"` } type K8SClusterInfo struct { @@ -53,26 +61,46 @@ type K8SClusterInfo struct { Memory string `json:"memory" bson:"memory"` } -func (K8SCluster) TableName() string { - return "k8s_cluster" +type AdvancedConfig struct { + Strategy string `json:"strategy,omitempty" bson:"strategy,omitempty"` + NodeLabels []*NodeSelectorRequirement `json:"node_labels,omitempty" bson:"node_labels,omitempty"` + ProjectNames []string `json:"-" bson:"-"` +} + +type NodeSelectorRequirement struct { + Key string `json:"key" bson:"key"` + Value []string `json:"value" bson:"value"` + Operator corev1.NodeSelectorOperator `json:"operator" bson:"operator"` +} + +type Limits struct { + CPU int `json:"cpu" bson:"cpu"` + Memory int `json:"memory" bson:"memory"` } -func (k *K8SCluster) Clean() error { - k.Name = strings.TrimSpace(k.Name) - tags := k.Tags - k.Tags = []string{} - for _, tag := range tags { - tag = strings.TrimSpace(tag) - if tag != "" { - k.Tags = append(k.Tags, tag) - } - } +type Resources struct { + Limits *Limits `json:"limits" bson:"limits"` +} - k.Description = strings.TrimSpace(k.Description) +type DindCfg struct { + Replicas int `json:"replicas" bson:"replicas"` + Resources *Resources `json:"resources" bson:"resources"` + Storage *DindStorage `json:"storage" bson:"storage"` +} - if !namePattern.MatchString(k.Name) { - return fmt.Errorf("集群名称不符合规则") - } +type DindStorageType string + +const ( + DindStorageRootfs DindStorageType = "rootfs" + DindStorageDynamic DindStorageType = "dynamic" +) - return nil +type DindStorage struct { + Type DindStorageType `json:"type" bson:"type"` + StorageClass string `json:"storage_class" bson:"storage_class"` + StorageSizeInGiB int64 `json:"storage_size_in_gib" bson:"storage_size_in_gib"` +} + +func (K8SCluster) TableName() string { + return "k8s_cluster" } diff --git a/pkg/microservice/aslan/core/common/repository/models/notification.go b/pkg/microservice/aslan/core/common/repository/models/notification.go index 7b33ee8dcdfe22107ca594b217fd3a60a12ec586..75f7a59c16dc6d9670b38b00c68d7c5b6cfa7fde 100644 --- a/pkg/microservice/aslan/core/common/repository/models/notification.go +++ b/pkg/microservice/aslan/core/common/repository/models/notification.go @@ -27,7 +27,7 @@ import ( ) type Notification struct { - ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` CodehostID int `bson:"codehost_id" json:"codehost_id"` Tasks []*NotificationTask `bson:"tasks,omitempty" json:"tasks,omitempty"` PrID int `bson:"pr_id" json:"pr_id"` @@ -37,10 +37,13 @@ type Notification struct { BaseURI string `bson:"base_uri" json:"base_uri"` IsPipeline bool `bson:"is_pipeline" json:"is_pipeline"` IsTest bool `bson:"is_test" json:"is_test"` + IsScanning bool `bson:"is_scanning" json:"is_scanning"` ErrInfo string `bson:"err_info" json:"err_info"` PrTask *PrTaskInfo `bson:"pr_task_info,omitempty" json:"pr_task_info,omitempty"` Label string `bson:"label" json:"label" ` Revision string `bson:"revision" json:"revision"` + RepoOwner string `bson:"repo_owner" json:"repo_owner"` + RepoName string `bson:"repo_name" json:"repo_name"` } type PrTaskInfo struct { @@ -54,6 +57,8 @@ type NotificationTask struct { ProductName string `bson:"product_name" json:"product_name"` WorkflowName string `bson:"workflow_name" json:"workflow_name"` PipelineName string `bson:"pipeline_name" json:"pipeline_name"` + ScanningName string `bson:"scanning_name" json:"scanningName"` + ScanningID string `bson:"scanning_id" json:"scanning_id"` TestName string `bson:"test_name" json:"test_name"` ID int64 `bson:"id" json:"id"` Status config.TaskStatus `bson:"status" json:"status"` @@ -121,6 +126,14 @@ func (n *Notification) CreateCommentBody() (comment string, err error) { tmplSource = "|触发的测试|状态|测试结果（成功数/总用例数量）| \n |---|---|---| \n {{range .Tasks}}|[{{.TestName}}#{{.ID}}]({{$.BaseURI}}/v1/projects/detail/{{.ProductName}}/test/detail/function/{{.TestName}}/{{.ID}}) | {{if eq .StatusVerbose $.Success}} {+ {{.StatusVerbose}} +}{{else}}{- {{.StatusVerbose}} -}{{end}} | {{range .TestReports}}{{.Name}}: {{.Successes}}/{{.Tests}}
{{end}} | \n {{end}}" } + } else if n.IsScanning { + if len(n.Tasks) == 0 { + tmplSource = "触发的代码扫描：等待任务启动中" + } else { + tmplSource = + "|触发的代码扫描|状态| \n |---|---| \n {{range .Tasks}}|[{{.ScanningName}}#{{.ID}}]({{$.BaseURI}}/v1/projects/detail/{{.ProductName}}/scanner/detail/{{.ScanningName}}/task/{{.ID}}?id={{.ScanningID}}) | {{if eq .StatusVerbose $.Success}} {+ {{.StatusVerbose}} +}{{else}}{- {{.StatusVerbose}} -}{{end}} | \n {{end}}" + + } } else { if len(n.Tasks) == 0 { tmplSource = "触发的工作流：等待任务启动中" diff --git a/pkg/microservice/aslan/core/common/repository/models/notify.go b/pkg/microservice/aslan/core/common/repository/models/notify.go index 88ae5d97bfb0599a372bb3d8db2a023895b7268d..af7c0e9b891b0410c46aac841fc58abce4a414d3 100644 --- a/pkg/microservice/aslan/core/common/repository/models/notify.go +++ b/pkg/microservice/aslan/core/common/repository/models/notify.go @@ -46,6 +46,7 @@ type PipelineStatusCtx struct { Type config.PipelineType `bson:"type" json:"type"` Status config.Status `bson:"status" json:"status,omitempty"` TeamName string `bson:"team" json:"team"` + Stages []*Stage `bson:"-" json:"stages"` } type MessageCtx struct { diff --git a/pkg/microservice/aslan/core/common/repository/models/pipeline.go b/pkg/microservice/aslan/core/common/repository/models/pipeline.go index 40cfe1561b54c960659c2860456cd7885e96aa68..960193e2084532cbc85a4fd3bbc84c8fc0e2dd77 100644 --- a/pkg/microservice/aslan/core/common/repository/models/pipeline.go +++ b/pkg/microservice/aslan/core/common/repository/models/pipeline.go @@ -32,8 +32,6 @@ type Pipeline struct { Enabled bool `bson:"enabled" json:"enabled"` TeamName string `bson:"team" json:"team"` ProductName string `bson:"product_name" json:"product_name"` - // OrgID 单租户ID - OrgID int `bson:"org_id" json:"org_id"` // target 服务名称, k8s为容器名称, 物理机为服务名 Target string `bson:"target" json:"target"` // 使用预定义编译管理模块中的内容生成SubTasks, @@ -53,8 +51,11 @@ type Pipeline struct { PassRate float32 `bson:"pass_rate,omitempty" json:"pass_rate,omitempty"` IsDeleted bool `bson:"is_deleted" json:"is_deleted"` Slack *Slack `bson:"slack" json:"slack"` - NotifyCtl *NotifyCtl `bson:"notify_ctl" json:"notify_ctl"` - IsFavorite bool `bson:"-" json:"is_favorite"` + // TODO: Deprecated. + NotifyCtl *NotifyCtl `bson:"notify_ctl" json:"notify_ctl"` + // New since V1.12.0. + NotifyCtls []*NotifyCtl `bson:"notify_ctls" json:"notify_ctls"` + IsFavorite bool `bson:"-" json:"is_favorite"` // 是否允许同时运行多次 MultiRun bool `bson:"multi_run" json:"multi_run"` } @@ -67,7 +68,7 @@ type Stage struct { RunParallel bool `bson:"run_parallel" json:"run_parallel"` Desc string `bson:"desc,omitempty" json:"desc,omitempty"` SubTasks map[string]map[string]interface{} `bson:"sub_tasks" json:"sub_tasks"` - AfterAll bool `json:"after_all" bson:"after_all"` + AfterAll bool `bson:"after_all" json:"after_all"` } type Hook struct { diff --git a/pkg/microservice/aslan/core/common/repository/models/plugin_repo.go b/pkg/microservice/aslan/core/common/repository/models/plugin_repo.go new file mode 100644 index 0000000000000000000000000000000000000000..20724c6642082fbcf7cbb4a6d0f4da2a1153ce83 --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/models/plugin_repo.go @@ -0,0 +1,71 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type PluginRepo struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty" yaml:"source,omitempty"` + IsOffical bool `bson:"is_offical" json:"is_offical" yaml:"is_offical"` + RepoURL string `bson:"repo_url" json:"repo_url" yaml:"-"` + RepoNamespace string `bson:"repo_namespace" json:"repo_namespace" yaml:"repo_namespace"` + Source string `bson:"source,omitempty" json:"source,omitempty" yaml:"source,omitempty"` + RepoOwner string `bson:"repo_owner" json:"repo_owner" yaml:"repo_owner"` + RepoName string `bson:"repo_name" json:"repo_name" yaml:"repo_name"` + Branch string `bson:"branch" json:"branch" yaml:"branch"` + CommitID string `bson:"commit_id,omitempty" json:"commit_id,omitempty" yaml:"commit_id,omitempty"` + CodehostID int `bson:"codehost_id" json:"codehost_id" yaml:"codehost_id"` + UpdateTime int64 `bson:"update_time" json:"update_time" yaml:"update_time"` + PluginTemplates []*PluginTemplate `bson:"plugin_templates" json:"plugin_templates" yaml:"plugin_templates"` + Status string `bson:"status" json:"status" yaml:"status"` + Error string `bson:"error" json:"error" yaml:"error"` +} + +type PluginTemplate struct { + Name string `bson:"name" json:"name" yaml:"name"` + IsOffical bool `bson:"is_offical" json:"is_offical" yaml:"is_offical"` + Description string `bson:"description" json:"description" yaml:"description"` + RepoURL string `bson:"repo_url" json:"repo_url" yaml:"-"` + Version string `bson:"version" json:"version" yaml:"version"` + Image string `bson:"image" json:"image" yaml:"image"` + Args []string `bson:"args" json:"args" yaml:"args"` + Cmds []string `bson:"cmds" json:"cmds" yaml:"cmds"` + Envs []*Env `bson:"envs" json:"envs" yaml:"envs"` + Inputs []*Params `bson:"inputs" json:"inputs" yaml:"inputs"` + Outputs []*Params `bson:"outputs" json:"outputs" yaml:"outputs"` +} + +type Env struct { + Name string `bson:"name" json:"name" yaml:"name"` + Value string `bson:"value" json:"value" yaml:"value"` +} + +type Params struct { + Name string `bson:"name" json:"name" yaml:"name"` + Description string `bson:"description" json:"description" yaml:"description"` + // support string/text type + ParamsType string `bson:"type" json:"type" yaml:"type"` + Value string `bson:"value" json:"value" yaml:"value,omitempty"` + Default string `bson:"default" json:"default" yaml:"default"` + IsCredential bool `bson:"is_credential" json:"is_credential" yaml:"is_credential"` +} + +func (PluginRepo) TableName() string { + return "plugin_repo" +} diff --git a/pkg/microservice/aslan/core/common/repository/models/private_key.go b/pkg/microservice/aslan/core/common/repository/models/private_key.go index f8e711bc4afda36fb2a6114cf1a3c257ae1c384c..2cfc60ebd8bb4ee5f59f4e49e2d4c647cb880682 100644 --- a/pkg/microservice/aslan/core/common/repository/models/private_key.go +++ b/pkg/microservice/aslan/core/common/repository/models/private_key.go @@ -17,33 +17,30 @@ limitations under the License. package models import ( + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/types" "go.mongodb.org/mongo-driver/bson/primitive" ) type PrivateKey struct { - ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` - Name string `bson:"name" json:"name"` - UserName string `bson:"user_name" json:"user_name"` - IP string `bson:"ip" json:"ip"` - Label string `bson:"label" json:"label"` - IsProd bool `bson:"is_prod" json:"is_prod"` - PrivateKey string `bson:"private_key" json:"private_key"` - CreateTime int64 `bson:"create_time" json:"create_time"` - UpdateTime int64 `bson:"update_time" json:"update_time"` - UpdateBy string `bson:"update_by" json:"update_by"` - Provider int8 `bson:"provider" json:"provider"` + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + Name string `bson:"name" json:"name"` + UserName string `bson:"user_name" json:"user_name"` + IP string `bson:"ip" json:"ip"` + Port int64 `bson:"port" json:"port"` + Status setting.PMHostStatus `bson:"status" json:"status"` + Label string `bson:"label" json:"label"` + IsProd bool `bson:"is_prod" json:"is_prod"` + PrivateKey string `bson:"private_key" json:"private_key"` + CreateTime int64 `bson:"create_time" json:"create_time"` + UpdateTime int64 `bson:"update_time" json:"update_time"` + UpdateBy string `bson:"update_by" json:"update_by"` + Provider int8 `bson:"provider" json:"provider"` + Probe *types.Probe `bson:"probe" json:"probe"` + ProjectName string `bson:"project_name,omitempty" json:"project_name"` + UpdateStatus bool `bson:"-" json:"update_status"` } func (PrivateKey) TableName() string { return "private_key" } - -//type SSH struct { -// ID string `json:"id"` -// Name string `json:"name"` -// UserName string `json:"user_name"` -// IP string `json:"ip"` -// IsProd bool `json:"is_prod"` -// Label string `json:"label"` -// PrivateKey string `json:"private_key"` -//} diff --git a/pkg/microservice/aslan/core/common/repository/models/product.go b/pkg/microservice/aslan/core/common/repository/models/product.go index 5bfa65c97f744ecfd1e18ba65234bfd2ea81ed7a..15d64a08e588a9cf920ab03c5bd7be8d35f2b283 100644 --- a/pkg/microservice/aslan/core/common/repository/models/product.go +++ b/pkg/microservice/aslan/core/common/repository/models/product.go @@ -19,6 +19,7 @@ package models import ( "go.mongodb.org/mongo-driver/bson/primitive" + "github.com/koderover/zadig/pkg/microservice/aslan/config" templatemodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" ) @@ -50,8 +51,33 @@ type Product struct { RecycleDay int `bson:"recycle_day" json:"recycle_day"` Source string `bson:"source" json:"source"` IsOpenSource bool `bson:"is_opensource" json:"is_opensource"` + RegistryID string `bson:"registry_id" json:"registry_id"` + BaseName string `bson:"base_name" json:"base_name"` + // IsExisted is true if this environment is created from an existing one + IsExisted bool `bson:"is_existed" json:"is_existed"` // TODO: temp flag IsForkedProduct bool `bson:"-" json:"-"` + + // New Since v1.11.0. + ShareEnv ProductShareEnv `bson:"share_env" json:"share_env"` + + // New Since v1.13.0. + EnvConfigs []*CreateUpdateCommonEnvCfgArgs `bson:"-" json:"env_configs,omitempty"` +} + +type CreateUpdateCommonEnvCfgArgs struct { + EnvName string `json:"env_name"` + ProductName string `json:"product_name"` + ServiceName string `json:"service_name"` + Name string `json:"name"` + YamlData string `json:"yaml_data"` + RestartAssociatedSvc bool `json:"restart_associated_svc"` + CommonEnvCfgType config.CommonEnvCfgType `json:"common_env_cfg_type"` + Services []string `json:"services"` + GitRepoConfig *templatemodels.GitRepoConfig `json:"git_repo_config"` + SourceDetail *CreateFromRepo `json:"-"` + AutoSync bool `json:"auto_sync"` + LatestEnvResource *EnvResource `json:"-"` } type RenderInfo struct { @@ -74,6 +100,7 @@ type ProductService struct { Revision int64 `bson:"revision" json:"revision"` Containers []*Container `bson:"containers" json:"containers,omitempty"` Render *RenderInfo `bson:"render,omitempty" json:"render,omitempty"` // 记录每个服务render信息 便于更新单个服务 + Error string `bson:"error,omitempty" json:"error,omitempty"` EnvConfigs []*EnvConfig `bson:"-" json:"env_configs,omitempty"` } @@ -82,6 +109,12 @@ type ServiceConfig struct { Revision int64 `bson:"revision" json:"revision"` } +type ProductShareEnv struct { + Enable bool `bson:"enable" json:"enable"` + IsBase bool `bson:"is_base" json:"is_base"` + BaseEnv string `bson:"base_env" json:"base_env"` +} + func (Product) TableName() string { return "product" } diff --git a/pkg/microservice/aslan/core/templatestore/repository/models/chart.go b/pkg/microservice/aslan/core/common/repository/models/project_cluster_relation.go similarity index 49% rename from pkg/microservice/aslan/core/templatestore/repository/models/chart.go rename to pkg/microservice/aslan/core/common/repository/models/project_cluster_relation.go index daf105ba3c44d62bc90eedad51d367a204a8f70e..7f6fd4226b047f5860402d95cd772e5fa0c29b34 100644 --- a/pkg/microservice/aslan/core/templatestore/repository/models/chart.go +++ b/pkg/microservice/aslan/core/common/repository/models/project_cluster_relation.go @@ -16,18 +16,18 @@ limitations under the License. package models -type Chart struct { - Name string `bson:"name" json:"name"` - Source string `bson:"source" json:"source"` - Owner string `bson:"owner" json:"owner"` - Repo string `bson:"repo" json:"repo"` - Path string `bson:"path" json:"path"` - Branch string `bson:"branch" json:"branch"` - CodeHostID int `bson:"codehost_id" json:"codeHostID"` - Revision int64 `bson:"revision" json:"revision"` - Sha1 string `bson:"sha1" json:"sha1"` +import ( + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type ProjectClusterRelation struct { + ID primitive.ObjectID `json:"id,omitempty" bson:"_id,omitempty"` + ProjectName string `json:"project_name" bson:"project_name"` + ClusterID string `json:"cluster_id" bson:"cluster_id"` + CreatedAt int64 `json:"createdAt" bson:"createdAt"` + CreatedBy string `json:"createdBy" bson:"createdBy"` } -func (Chart) TableName() string { - return "chart_template" +func (ProjectClusterRelation) TableName() string { + return "project_cluster_relation" } diff --git a/pkg/microservice/aslan/core/common/repository/models/queue.go b/pkg/microservice/aslan/core/common/repository/models/queue.go index d6279a8b30134c3f1c1402f425324f434a712e01..826120ba9fcd68c1b4aa736c8b0053c734b004f6 100644 --- a/pkg/microservice/aslan/core/common/repository/models/queue.go +++ b/pkg/microservice/aslan/core/common/repository/models/queue.go @@ -22,80 +22,68 @@ import ( "go.mongodb.org/mongo-driver/bson/primitive" "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/setting" ) type Queue struct { - ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` - TaskID int64 `bson:"task_id" json:"task_id"` - ProductName string `bson:"product_name" json:"product_name"` - PipelineName string `bson:"pipeline_name" json:"pipeline_name"` - Namespace string `bson:"namespace,omitempty" json:"namespace,omitempty"` - Type config.PipelineType `bson:"type" json:"type"` - Status config.Status `bson:"status" json:"status,omitempty"` - Description string `bson:"description,omitempty" json:"description,omitempty"` - TaskCreator string `bson:"task_creator" json:"task_creator,omitempty"` - TaskRevoker string `bson:"task_revoker,omitempty" json:"task_revoker,omitempty"` - CreateTime int64 `bson:"create_time" json:"create_time,omitempty"` - StartTime int64 `bson:"start_time" json:"start_time,omitempty"` - EndTime int64 `bson:"end_time" json:"end_time,omitempty"` - SubTasks []map[string]interface{} `bson:"sub_tasks" json:"sub_tasks"` - Stages []*Stage `bson:"stages" json:"stages"` - ReqID string `bson:"req_id,omitempty" json:"req_id,omitempty"` - AgentHost string `bson:"agent_host,omitempty" json:"agent_host,omitempty"` - DockerHost string `bson:"-" json:"docker_host,omitempty"` - TeamID int `bson:"team_id,omitempty" json:"team_id,omitempty"` - TeamName string `bson:"team,omitempty" json:"team,omitempty"` - IsDeleted bool `bson:"is_deleted" json:"is_deleted"` - IsArchived bool `bson:"is_archived" json:"is_archived"` - AgentID string `bson:"agent_id" json:"agent_id"` - // 是否允许同时运行多次 - MultiRun bool `bson:"multi_run" json:"multi_run"` - // target 服务名称, k8s为容器名称, 物理机为服务名 - Target string `bson:"target,omitempty" json:"target"` - // 使用预定义编译管理模块中的内容生成SubTasks, - // 查询条件为 服务模板名称: ServiceTmpl, 版本: BuildModuleVer - // 如果为空，则使用pipeline自定义SubTasks - BuildModuleVer string `bson:"build_module_ver,omitempty" json:"build_module_ver"` - ServiceName string `bson:"service_name,omitempty" json:"service_name,omitempty"` - // TaskArgs 单服务工作流任务参数 - TaskArgs *TaskArgs `bson:"task_args,omitempty" json:"task_args,omitempty"` - // WorkflowArgs 多服务工作流任务参数 - WorkflowArgs *WorkflowTaskArgs `bson:"workflow_args" json:"workflow_args,omitempty"` - // TestArgs 测试任务参数 - TestArgs *TestTaskArgs `bson:"test_args,omitempty" json:"test_args,omitempty"` - // ServiceTaskArgs 脚本部署工作流任务参数 - ServiceTaskArgs *ServiceTaskArgs `bson:"service_args,omitempty" json:"service_args,omitempty"` - ConfigPayload *ConfigPayload `json:"config_payload,omitempty"` - Error string `bson:"error,omitempty" json:"error,omitempty"` - // OrgID 单租户ID - OrgID int `bson:"org_id,omitempty" json:"org_id,omitempty"` - Services [][]*ProductService `bson:"services" json:"services"` - Render *RenderInfo `bson:"render" json:"render"` - StorageURI string `bson:"storage_uri,omitempty" json:"storage_uri,omitempty"` - // interface{} 为types.TestReport - TestReports map[string]interface{} `bson:"test_reports,omitempty" json:"test_reports,omitempty"` - - RwLock sync.Mutex `bson:"-" json:"-"` - - ResetImage bool `json:"resetImage" bson:"resetImage"` - - TriggerBy *TriggerBy `json:"trigger_by,omitempty" bson:"trigger_by,omitempty"` - - Features []string `bson:"features" json:"features"` - IsRestart bool `bson:"is_restart" json:"is_restart"` - StorageEndpoint string `bson:"storage_endpoint" json:"storage_endpoint"` + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + TaskID int64 `bson:"task_id" json:"task_id"` + ProductName string `bson:"product_name" json:"product_name"` + PipelineName string `bson:"pipeline_name" json:"pipeline_name"` + Namespace string `bson:"namespace,omitempty" json:"namespace,omitempty"` + Type config.PipelineType `bson:"type" json:"type"` + Status config.Status `bson:"status" json:"status,omitempty"` + Description string `bson:"description,omitempty" json:"description,omitempty"` + TaskCreator string `bson:"task_creator" json:"task_creator,omitempty"` + TaskRevoker string `bson:"task_revoker,omitempty" json:"task_revoker,omitempty"` + CreateTime int64 `bson:"create_time" json:"create_time,omitempty"` + StartTime int64 `bson:"start_time" json:"start_time,omitempty"` + EndTime int64 `bson:"end_time" json:"end_time,omitempty"` + SubTasks []map[string]interface{} `bson:"sub_tasks" json:"sub_tasks"` + Stages []*Stage `bson:"stages" json:"stages"` + ReqID string `bson:"req_id,omitempty" json:"req_id,omitempty"` + AgentHost string `bson:"agent_host,omitempty" json:"agent_host,omitempty"` + DockerHost string `bson:"-" json:"docker_host,omitempty"` + TeamID int `bson:"team_id,omitempty" json:"team_id,omitempty"` + TeamName string `bson:"team,omitempty" json:"team,omitempty"` + IsDeleted bool `bson:"is_deleted" json:"is_deleted"` + IsArchived bool `bson:"is_archived" json:"is_archived"` + AgentID string `bson:"agent_id" json:"agent_id"` + MultiRun bool `bson:"multi_run" json:"multi_run"` + Target string `bson:"target,omitempty" json:"target"` // target service name, for k8s: containerName, for pm: serviceName + BuildModuleVer string `bson:"build_module_ver,omitempty" json:"build_module_ver"` + ServiceName string `bson:"service_name,omitempty" json:"service_name,omitempty"` + TaskArgs *TaskArgs `bson:"task_args,omitempty" json:"task_args,omitempty"` // TaskArgs job parameters for single-service workflow + WorkflowArgs *WorkflowTaskArgs `bson:"workflow_args" json:"workflow_args,omitempty"` // WorkflowArgs job parameters for multi-service workflow + TestArgs *TestTaskArgs `bson:"test_args,omitempty" json:"test_args,omitempty"` // TestArgs parameters for testing + ServiceTaskArgs *ServiceTaskArgs `bson:"service_args,omitempty" json:"service_args,omitempty"` // ServiceTaskArgs parameters for script-deployed workflows + ArtifactPackageTaskArgs *ArtifactPackageTaskArgs `bson:"artifact_package_args,omitempty" json:"artifact_package_args,omitempty"` + ConfigPayload *ConfigPayload `bson:"configpayload,omitempty" json:"config_payload"` + Error string `bson:"error,omitempty" json:"error,omitempty"` + Services [][]*ProductService `bson:"services" json:"services"` + Render *RenderInfo `bson:"render" json:"render"` + StorageURI string `bson:"storage_uri,omitempty" json:"storage_uri,omitempty"` + TestReports map[string]interface{} `bson:"test_reports,omitempty" json:"test_reports,omitempty"` + RwLock sync.Mutex `bson:"-" json:"-"` + ResetImage bool `bson:"resetImage" json:"resetImage"` + ResetImagePolicy setting.ResetImagePolicyType `bson:"reset_image_policy" json:"reset_image_policy"` + TriggerBy *TriggerBy `bson:"trigger_by,omitempty" json:"trigger_by,omitempty"` + Features []string `bson:"features" json:"features"` + IsRestart bool `bson:"is_restart" json:"is_restart"` + StorageEndpoint string `bson:"storage_endpoint" json:"storage_endpoint"` } type TriggerBy struct { // 触发此次任务的代码库信息 - CodehostID int `bson:"codehost_id,omitempty" json:"codehost_id,omitempty"` - RepoOwner string `bson:"repo_owner,omitempty" json:"repo_owner,omitempty"` - RepoName string `bson:"repo_name,omitempty" json:"repo_name,omitempty"` - Source string `json:"source,omitempty" bson:"source,omitempty"` + CodehostID int `bson:"codehost_id,omitempty" json:"codehost_id,omitempty"` + RepoOwner string `bson:"repo_owner,omitempty" json:"repo_owner,omitempty"` + RepoName string `bson:"repo_name,omitempty" json:"repo_name,omitempty"` + RepoNamespace string `bson:"repo_namespace" json:"repo_namespace"` + Source string `json:"source,omitempty" bson:"source,omitempty"` // 触发此次任务的merge request id，用于判断多个任务对应的commit是否属于同一个merge request MergeRequestID string `json:"merge_request_id,omitempty" bson:"merge_request_id,omitempty"` // 触发此次任务的commit id - CommitID string `json:"commit_id,omitempty" bson:"commit_id,omitempty"` + CommitID string `bson:"commit_id,omitempty" json:"commit_id,omitempty"` } type ServiceTaskArgs struct { @@ -110,8 +98,28 @@ type ServiceTaskArgs struct { Updatable bool `bson:"updatable" json:"updatable"` } +type ImageData struct { + ImageUrl string `bson:"image_url" json:"image_url"` + ImageName string `bson:"image_name" json:"image_name"` + ImageTag string `bson:"image_tag" json:"image_tag"` + CustomTag string `bson:"custom_tag" json:"custom_tag"` + RegistryID string `bson:"registry_id" json:"registry_id"` +} + +type ImagesByService struct { + ServiceName string `bson:"service_name" json:"service_name"` + Images []*ImageData `bson:"images" json:"images"` +} + +type ArtifactPackageTaskArgs struct { + ProjectName string `bson:"project_name" json:"project_name"` + EnvName string `bson:"env_name" json:"env_name"` + Images []*ImagesByService `bson:"images" json:"images"` + SourceRegistries []string `bson:"source_registries" json:"source_registries"` + TargetRegistries []string `bson:"target_registries" json:"target_registries"` +} + type ConfigPayload struct { - Aslan AslanConfig `json:"aslan"` Proxy Proxy `json:"proxy"` S3Storage S3Config `json:"s3_storage"` Github GithubConfig `json:"github"` @@ -129,6 +137,7 @@ type ConfigPayload struct { CustomDNSSupported bool `json:"custom_dns_supported"` HubServerAddr string DeployClusterID string + AesKey string `json:"aes_key"` RepoConfigs map[string]*RegistryNamespace @@ -139,6 +148,13 @@ type ConfigPayload struct { ResetCache bool `json:"reset_cache"` JenkinsBuildConfig JenkinsBuildConfig `json:"jenkins_build_config"` PrivateKeys []*PrivateKey `json:"private_keys"` + K8SClusters []*K8SClusterResp `json:"k8s_clusters"` + + // RegistryID is the id of product registry + RegistryID string `json:"registry_id"` + + // build concurrency settings + BuildConcurrency int64 `json:"build_concurrency"` } type AslanConfig struct { @@ -164,15 +180,15 @@ type S3Config struct { type GithubConfig struct { // github API access token - AccessToken string + AccessToken string `json:"access_token"` // github ssh key with base64 encoded - SSHKey string + SSHKey string `json:"ssh_key"` // github knownhost - KnownHost string + KnownHost string `json:"known_host"` // github app private key - AppKey string + AppKey string `json:"app_key"` // gihhub app id - AppID int + AppID int `json:"app_id"` } type GitlabConfig struct { @@ -203,14 +219,17 @@ type RegistryConfig struct { type ReleaseConfig struct { // ReaperImage sets build job image - // e.g. xxx.com/poetry-resources/reaper-plugin:1.0.0 + // e.g. xxx.com/resources/reaper-plugin:1.0.0 ReaperImage string // ReaperBinaryFile sets download url of reaper binary file in build job // e.g. http://resource.koderover.com/reaper-20201014203000 ReaperBinaryFile string // PredatorImage sets docker build image - // e.g. xxx.com/poetry-resources/predator-plugin:v0.1.0 + // e.g. xxx.com/resources/predator-plugin:v0.1.0 PredatorImage string + // PackagerImage sets docker build image + // e.g. xxx.com/resources/predator-plugin:v0.1.0 + PackagerImage string } type ImageReleaseConfig struct { diff --git a/pkg/microservice/aslan/core/common/repository/models/registry_namespace.go b/pkg/microservice/aslan/core/common/repository/models/registry_namespace.go index 3c45ae9848660e67d7776c98d23147221add43be..388f0b34de1159373b985eb1ac208b262fb52d44 100644 --- a/pkg/microservice/aslan/core/common/repository/models/registry_namespace.go +++ b/pkg/microservice/aslan/core/common/repository/models/registry_namespace.go @@ -20,27 +20,39 @@ import ( "errors" "go.mongodb.org/mongo-driver/bson/primitive" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" ) type RegistryNamespace struct { ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` - OrgID int `bson:"org_id" json:"org_id"` RegAddr string `bson:"reg_addr" json:"reg_addr"` RegType string `bson:"reg_type" json:"reg_type"` RegProvider string `bson:"reg_provider" json:"reg_provider"` IsDefault bool `bson:"is_default" json:"is_default"` - Namespace string `bson:"namespace" json:"namespace"` - AccessKey string `bson:"access_key" json:"access_key"` - SecretKey string `bson:"secret_key" json:"secret_key"` - Region string `bson:"region,omitempty" json:"region,omitempty"` - UpdateTime int64 `bson:"update_time" json:"update_time"` - UpdateBy string `bson:"update_by" json:"update_by"` + // Namespace is NOT a required field, this could be empty when the registry is AWS ECR or so. + // use with CAUTION !!!! + Namespace string `bson:"namespace,omitempty" json:"namespace,omitempty"` + AccessKey string `bson:"access_key" json:"access_key"` + SecretKey string `bson:"secret_key" json:"secret_key"` + Region string `bson:"region,omitempty" json:"region,omitempty"` + UpdateTime int64 `bson:"update_time" json:"update_time"` + UpdateBy string `bson:"update_by" json:"update_by"` + + AdvancedSetting *RegistryAdvancedSetting `bson:"advanced_setting" json:"advanced_setting"` +} + +type RegistryAdvancedSetting struct { + // indicator to make sure if the advanced has been modified + // This is solely a field for frontend display and not used anywhere else in the backend system + // TODO: whether this field should exist needs a discussion + Modified bool `bson:"modified" json:"modified"` + // New field since v1.11 to support self-signed TLS certificate + TLSEnabled bool `bson:"enable_tls" json:"enable_tls"` + TLSCert string `bson:"tls_cert" json:"tls_cert"` } func (ns *RegistryNamespace) Validate() error { - if ns.OrgID == 0 { - return errors.New("empty org_id") - } if ns.RegAddr == "" { return errors.New("empty reg_addr") @@ -50,7 +62,8 @@ func (ns *RegistryNamespace) Validate() error { return errors.New("empty reg_provider") } - if ns.Namespace == "" { + // if the registry type is aws ECR, then the RegAddr is the registry's full + if ns.Namespace == "" && ns.RegProvider != config.RegistryTypeAWS { return errors.New("empty namespace") } @@ -60,7 +73,3 @@ func (ns *RegistryNamespace) Validate() error { func (RegistryNamespace) TableName() string { return "registry_namespace" } - -//const CandidateRegType = "candidate" -// -//const DistRegType = "dist" diff --git a/pkg/microservice/aslan/core/common/repository/models/render_set.go b/pkg/microservice/aslan/core/common/repository/models/render_set.go index 5899120f2da88831e84041dac3b39c5005ca74ad..158146708c506599df1e9a81dfb69a63b082c600 100644 --- a/pkg/microservice/aslan/core/common/repository/models/render_set.go +++ b/pkg/microservice/aslan/core/common/repository/models/render_set.go @@ -28,15 +28,17 @@ type RenderSet struct { Name string `bson:"name" json:"name"` Revision int64 `bson:"revision" json:"revision"` // 可以为空，空时为产品模板默认的渲染集，非空时为环境的渲染集 - EnvName string `bson:"env_name,omitempty" json:"env_name,omitempty"` - ProductTmpl string `bson:"product_tmpl" json:"product_tmpl"` - Team string `bson:"team,omitempty" json:"team,omitempty"` - UpdateTime int64 `bson:"update_time" json:"update_time"` - UpdateBy string `bson:"update_by" json:"update_by"` - IsDefault bool `bson:"is_default" json:"is_default"` // 是否是默认配置 - KVs []*templatemodels.RenderKV `bson:"kvs,omitempty" json:"kvs,omitempty"` - ChartInfos []*templatemodels.RenderChart `bson:"chart_infos,omitempty" json:"chart_infos,omitempty"` - Description string `bson:"description,omitempty" json:"description,omitempty"` + EnvName string `bson:"env_name,omitempty" json:"env_name,omitempty"` + ProductTmpl string `bson:"product_tmpl" json:"product_tmpl"` + Team string `bson:"team,omitempty" json:"team,omitempty"` + UpdateTime int64 `bson:"update_time" json:"update_time"` + UpdateBy string `bson:"update_by" json:"update_by"` + IsDefault bool `bson:"is_default" json:"is_default"` + DefaultValues string `bson:"default_values,omitempty" json:"default_values,omitempty"` + YamlData *templatemodels.CustomYaml `bson:"yaml_data,omitempty" json:"yaml_data,omitempty"` + KVs []*templatemodels.RenderKV `bson:"kvs,omitempty" json:"kvs,omitempty"` + ChartInfos []*templatemodels.RenderChart `bson:"chart_infos,omitempty" json:"chart_infos,omitempty"` + Description string `bson:"description,omitempty" json:"description,omitempty"` } func (RenderSet) TableName() string { diff --git a/pkg/microservice/aslan/core/common/repository/models/s3.go b/pkg/microservice/aslan/core/common/repository/models/s3.go index 23a2be10e093b132ab9e75bfd304b6707086085b..c0381b275e2804edf80bbc09ec89f70d901bc30f 100644 --- a/pkg/microservice/aslan/core/common/repository/models/s3.go +++ b/pkg/microservice/aslan/core/common/repository/models/s3.go @@ -35,6 +35,14 @@ type S3Storage struct { Provider int8 `bson:"provider" json:"provider"` } +type TarInfo struct { + URL string `bson:"url" json:"url"` + Name string `bson:"name" json:"name"` + WorkflowName string `bson:"workflow_name" json:"workflow_name"` + TaskID int64 `bson:"task_id" json:"task_id"` + FileName string `bson:"file_name" json:"file_name"` +} + func (s S3Storage) TableName() string { return "s3storage" } diff --git a/pkg/microservice/aslan/core/common/repository/models/scanning.go b/pkg/microservice/aslan/core/common/repository/models/scanning.go new file mode 100644 index 0000000000000000000000000000000000000000..b18601e07592de956a94ca1d8edd4970363e4882 --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/models/scanning.go @@ -0,0 +1,47 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" + + "github.com/koderover/zadig/pkg/types" +) + +type Scanning struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + Name string `bson:"name" json:"name"` + ProjectName string `bson:"project_name" json:"project_name"` + Description string `bson:"description" json:"description"` + ScannerType string `bson:"scanner_type" json:"scanner_type"` + ImageID string `bson:"image_id" json:"image_id"` + SonarID string `bson:"sonar_id" json:"sonar_id"` + Repos []*types.Repository `bson:"repos" json:"repos"` + // Parameter is for sonarQube type only + Parameter string `bson:"parameter" json:"parameter"` + // Script is for other type only + Script string `bson:"script" json:"script"` + AdvancedSetting *types.ScanningAdvancedSetting `bson:"advanced_setting" json:"advanced_setting"` + + CreatedAt int64 `bson:"created_at" json:"created_at"` + UpdatedAt int64 `bson:"updated_at" json:"updated_at"` + UpdatedBy string `bson:"updated_by" json:"updated_by"` +} + +func (Scanning) TableName() string { + return "scanning" +} diff --git a/pkg/microservice/aslan/core/common/repository/models/service.go b/pkg/microservice/aslan/core/common/repository/models/service.go index a5004f2d95f5920b21a146196c3d7b20e1238f0e..8a1541598d614d417636d8b701e16b657587b934 100644 --- a/pkg/microservice/aslan/core/common/repository/models/service.go +++ b/pkg/microservice/aslan/core/common/repository/models/service.go @@ -18,6 +18,8 @@ package models import ( templatemodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" + "github.com/koderover/zadig/pkg/setting" + "go.mongodb.org/mongo-driver/bson/primitive" ) // TODO: move Revision out of Service. @@ -50,25 +52,30 @@ type Service struct { GerritRemoteName string `bson:"gerrit_remote_name,omitempty" json:"gerrit_remote_name,omitempty"` GerritPath string `bson:"gerrit_path,omitempty" json:"gerrit_path,omitempty"` GerritCodeHostID int `bson:"gerrit_codeHost_id,omitempty" json:"gerrit_codeHost_id,omitempty"` - BuildName string `bson:"build_name,omitempty" json:"build_name,omitempty"` + GiteePath string `bson:"gitee_path,omitempty" json:"gitee_path,omitempty"` + BuildName string `bson:"build_name" json:"build_name"` HelmChart *HelmChart `bson:"helm_chart,omitempty" json:"helm_chart,omitempty"` EnvConfigs []*EnvConfig `bson:"env_configs,omitempty" json:"env_configs,omitempty"` EnvStatuses []*EnvStatus `bson:"env_statuses,omitempty" json:"env_statuses,omitempty"` + ReleaseNaming string `bson:"release_naming" json:"release_naming"` CodehostID int `bson:"codehost_id,omitempty" json:"codehost_id,omitempty"` RepoOwner string `bson:"repo_owner,omitempty" json:"repo_owner,omitempty"` + RepoNamespace string `bson:"repo_namespace,omitempty" json:"repo_namespace,omitempty"` RepoName string `bson:"repo_name,omitempty" json:"repo_name,omitempty"` RepoUUID string `bson:"repo_uuid,omitempty" json:"repo_uuid,omitempty"` BranchName string `bson:"branch_name,omitempty" json:"branch_name,omitempty"` LoadPath string `bson:"load_path,omitempty" json:"load_path,omitempty"` LoadFromDir bool `bson:"is_dir,omitempty" json:"is_dir,omitempty"` - CreateFrom interface{} `bson:"create_from,omitempty" json:"create_from,omitempty"` + CreateFrom interface{} `bson:"create_from,omitempty" json:"create_from,omitempty"` HealthChecks []*PmHealthCheck `bson:"health_checks,omitempty" json:"health_checks,omitempty"` WorkloadType string `bson:"workload_type,omitempty" json:"workload_type,omitempty"` EnvName string `bson:"env_name,omitempty" json:"env_name,omitempty"` + TemplateID string `bson:"template_id,omitempty" json:"template_id,omitempty"` + AutoSync bool `bson:"auto_sync" json:"auto_sync"` } type CreateFromRepo struct { - GitRepoConfig *templatemodels.GitRepoConfig `bson:"git_repo_config,omitempty" json:"git_repo_config,omitempty"` + GitRepoConfig *templatemodels.GitRepoConfig `bson:"git_repo_config,omitempty" json:"git_repo_config,omitempty"` LoadPath string `bson:"load_path,omitempty" json:"load_path,omitempty"` } @@ -81,6 +88,18 @@ type CreateFromChartTemplate struct { YamlData *templatemodels.CustomYaml `bson:"yaml_data,omitempty" json:"yaml_data,omitempty"` TemplateName string `bson:"template_name" json:"template_name"` ServiceName string `bson:"service_name" json:"service_name"` + Variables []*Variable `bson:"variables" json:"variables"` +} + +type CreateFromChartRepo struct { + ChartRepoName string `json:"chart_repo_name" bson:"chart_repo_name"` + ChartName string `json:"chart_name" bson:"chart_name"` + ChartVersion string `json:"chart_version" bson:"chart_version"` +} + +type CreateFromYamlTemplate struct { + TemplateID string `bson:"template_id" json:"template_id"` + Variables []*Variable `bson:"variables" json:"variables"` } type GUIConfig struct { @@ -122,8 +141,9 @@ type ImagePathSpec struct { // Container ... type Container struct { - Name string `bson:"name" json:"name"` - Image string `bson:"image" json:"image"` + Name string `bson:"name" json:"name"` + Image string `bson:"image" json:"image"` + ImageName string `bson:"image_name,omitempty" json:"image_name,omitempty"` ImagePath *ImagePathSpec `bson:"image_path,omitempty" json:"imagePath,omitempty"` } @@ -212,15 +232,28 @@ type HelmServiceRespArgs struct { } type EnvStatus struct { - HostID string `bson:"host_id,omitempty" json:"host_id"` - EnvName string `bson:"env_name,omitempty" json:"env_name"` - Address string `bson:"address,omitempty" json:"address"` - Status string `bson:"status,omitempty" json:"status"` + HostID string `bson:"host_id,omitempty" json:"host_id"` + EnvName string `bson:"env_name,omitempty" json:"env_name"` + Address string `bson:"address,omitempty" json:"address"` + Status string `bson:"status,omitempty" json:"status"` + PmInfo *PmInfo `bson:"-" json:"pm_info"` +} + +type PmInfo struct { + ID primitive.ObjectID `json:"id,omitempty"` + Name string `json:"name"` + IP string `json:"ip"` + Port int64 `json:"port"` + Status setting.PMHostStatus `json:"status"` + Label string `json:"label"` + IsProd bool `json:"is_prod"` + Provider int8 `json:"provider"` } type EnvConfig struct { EnvName string `bson:"env_name,omitempty" json:"env_name"` HostIDs []string `bson:"host_ids,omitempty" json:"host_ids"` + Labels []string `bson:"labels,omitempty" json:"labels"` } type PmHealthCheck struct { @@ -235,6 +268,20 @@ type PmHealthCheck struct { CurrentUnhealthyNum int `bson:"current_unhealthy_num,omitempty" json:"current_unhealthy_num,omitempty"` } +func (svc *Service) GetRepoNamespace() string { + if svc.RepoNamespace != "" { + return svc.RepoNamespace + } + return svc.RepoOwner +} + +func (svc *Service) GetReleaseNaming() string { + if len(svc.ReleaseNaming) > 0 { + return svc.ReleaseNaming + } + return setting.ReleaseNamingPlaceholder +} + func (Service) TableName() string { return "template_service" } diff --git a/pkg/microservice/aslan/core/common/repository/models/services_in_external_env.go b/pkg/microservice/aslan/core/common/repository/models/services_in_external_env.go new file mode 100644 index 0000000000000000000000000000000000000000..5320e189bb6538c40aae3303bfb8b79e44b8cad3 --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/models/services_in_external_env.go @@ -0,0 +1,29 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +type ServicesInExternalEnv struct { + ProductName string `bson:"product_name" json:"product_name"` + ServiceName string `bson:"service_name" json:"service_name"` + EnvName string `bson:"env_name" json:"env_name"` + Namespace string `bson:"namespace" json:"namespace"` + ClusterID string `bson:"cluster_id" json:"cluster_id"` +} + +func (ServicesInExternalEnv) TableName() string { + return "services_in_external_env" +} diff --git a/pkg/microservice/aslan/core/common/repository/models/settings.go b/pkg/microservice/aslan/core/common/repository/models/settings.go new file mode 100644 index 0000000000000000000000000000000000000000..2a490c9559995fc1a1ab3bc5c508b7b4603a77e5 --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/models/settings.go @@ -0,0 +1,31 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type SystemSetting struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + WorkflowConcurrency int64 `bson:"workflow_concurrency" json:"workflow_concurrency"` + BuildConcurrency int64 `bson:"build_concurrency" json:"build_concurrency"` + DefaultLogin string `bson:"default_login" json:"default_login"` + UpdateTime int64 `bson:"update_time" json:"update_time"` +} + +func (SystemSetting) TableName() string { + return "system_setting" +} diff --git a/pkg/microservice/aslan/core/common/repository/models/sonar.go b/pkg/microservice/aslan/core/common/repository/models/sonar.go new file mode 100644 index 0000000000000000000000000000000000000000..e81499c1c1f155e560918d6b38fc897d13aa6b4c --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/models/sonar.go @@ -0,0 +1,29 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type SonarIntegration struct { + ID primitive.ObjectID `bson:"_id,omitempty"` + ServerAddress string `bson:"server_address"` + Token string `bson:"token"` +} + +func (SonarIntegration) TableName() string { + return "sonar_integration" +} diff --git a/pkg/microservice/aslan/core/common/repository/models/strategy.go b/pkg/microservice/aslan/core/common/repository/models/strategy.go index ed76876e3efc8cbb4efb5c51635e276a76f6b888..7e28ccb41f56d46ffcbbbc2a3022c12452b969d5 100644 --- a/pkg/microservice/aslan/core/common/repository/models/strategy.go +++ b/pkg/microservice/aslan/core/common/repository/models/strategy.go @@ -23,9 +23,17 @@ type CapacityTarget string const ( // 工作流任务的留存 - WorkflowTaskRetention CapacityTarget = "WorkflowTaskRetention" + WorkflowTaskRetention CapacityTarget = "WorkflowTaskRetention" + DefaultWorkflowRemainDays int = 365 ) +var DefaultWorkflowTaskRetention = &CapacityStrategy{ + Target: WorkflowTaskRetention, + Retention: &RetentionConfig{ + MaxDays: DefaultWorkflowRemainDays, + }, +} + // RetentionConfig 资源留存相关的配置 type RetentionConfig struct { MaxDays int `bson:"max_days" json:"max_days"` // 最多几天 diff --git a/pkg/microservice/aslan/core/common/repository/models/task/artifact_package.go b/pkg/microservice/aslan/core/common/repository/models/task/artifact_package.go new file mode 100644 index 0000000000000000000000000000000000000000..89f89de42a98c629063640d9a351b938fd3bf2db --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/models/task/artifact_package.go @@ -0,0 +1,75 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package task + +import ( + "encoding/json" + "fmt" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" +) + +type ImageData struct { + ImageUrl string `bson:"image_url" json:"image_url"` + ImageName string `bson:"image_name" json:"image_name"` + ImageNamespace string `bson:"image_namespace" json:"image_namespace"` + ImageTag string `bson:"image_tag" json:"image_tag"` +} + +type ServicePackageResult struct { + ServiceName string `json:"service_name"` + Result string `json:"result"` + ErrorMsg string `json:"error_msg"` + ImageData []*ImageData `json:"image_data"` +} + +type ArtifactPackage struct { + TaskType config.TaskType `bson:"type" json:"type"` + TaskStatus config.Status `bson:"status" json:"status"` + Enabled bool `bson:"enabled" json:"enabled"` + Timeout int `bson:"timeout,omitempty" json:"timeout,omitempty"` + Error string `bson:"error,omitempty" json:"error,omitempty"` + StartTime int64 `bson:"start_time,omitempty" json:"start_time,omitempty"` + EndTime int64 `bson:"end_time,omitempty" json:"end_time,omitempty"` + LogFile string `bson:"log_file" json:"log_file"` + Progress string `bson:"progress" json:"progress"` + + // source images + Images []*models.ImagesByService `bson:"images" json:"images"` + // source registries need auth to pull images + SourceRegistries []string `bson:"source_registries" json:"source_registries"` + // target registries to push images + TargetRegistries []string `bson:"target_registries" json:"target_registries"` +} + +func (ri *ArtifactPackage) ToSubTask() (map[string]interface{}, error) { + var task map[string]interface{} + if err := IToi(ri, &task); err != nil { + return nil, fmt.Errorf("convert ReleaseImageTask to interface error: %v", err) + } + return task, nil +} + +func (ri *ArtifactPackage) GetProgress() ([]*ServicePackageResult, error) { + if len(ri.Progress) == 0 { + return nil, nil + } + ret := make([]*ServicePackageResult, 0) + err := json.Unmarshal([]byte(ri.Progress), &ret) + return ret, err +} diff --git a/pkg/microservice/aslan/core/common/repository/models/task/build.go b/pkg/microservice/aslan/core/common/repository/models/task/build.go index 47f18074907221e1e33aacbfa08bc20fab5695d0..96b4dff86e0e967f2afa18f3259dda3bd64c104e 100644 --- a/pkg/microservice/aslan/core/common/repository/models/task/build.go +++ b/pkg/microservice/aslan/core/common/repository/models/task/build.go @@ -26,9 +26,10 @@ import ( ) type Build struct { - TaskType config.TaskType `bson:"type" json:"type"` - Enabled bool `bson:"enabled" json:"enabled"` - TaskStatus config.Status `bson:"status" json:"status"` + TaskType config.TaskType `bson:"type" json:"type"` + Enabled bool `bson:"enabled" json:"enabled"` + TaskStatus config.Status `bson:"status" json:"status"` + ProductName string `bson:"product_name" json:"product_name"` // 新增一个service表示服务名称 Service string `bson:"service" json:"service"` // 该名称实际为服务组件名称 @@ -47,6 +48,7 @@ type Build struct { ImageFrom string `bson:"image_from" json:"image_from,omitempty"` ImageID string `bson:"image_id" json:"image_id"` ResReq setting.Request `bson:"res_req" json:"res_req"` + ResReqSpec setting.RequestSpec `bson:"res_req_spec" json:"res_req_spec"` LogFile string `bson:"log_file" json:"log_file"` InstallCtx []*models.Install `bson:"-" json:"install_ctx,omitempty"` Registries []*models.RegistryNamespace `bson:"-" json:"registries"` @@ -56,7 +58,23 @@ type Build struct { BuildStatus *BuildStatus `bson:"build_status,omitempty" json:"build_status,omitempty"` IsRestart bool `bson:"is_restart" json:"is_restart"` // Get the host bound to the environment of the cloud host service configuration - EnvHostInfo map[string][]string `bson:"env_host_info,omitempty" json:"env_host_info,omitempty"` + EnvHostInfo map[string][]string `bson:"env_host_info,omitempty" json:"env_host_info,omitempty"` + EnvHostNames map[string][]string `bson:"env_host_names,omitempty" json:"env_host_names,omitempty"` + ArtifactInfo *ArtifactInfo `bson:"artifact_info,omitempty" json:"artifact_info,omitempty"` + ClusterID string `bson:"cluster_id,omitempty" json:"cluster_id,omitempty"` + + // New since V1.10.0. + Cache types.Cache `bson:"cache" json:"cache"` + CacheEnable bool `bson:"cache_enable" json:"cache_enable"` + CacheDirType types.CacheDirType `bson:"cache_dir_type" json:"cache_dir_type"` + CacheUserDir string `bson:"cache_user_dir" json:"cache_user_dir"` +} + +type ArtifactInfo struct { + URL string `bson:"url" json:"url"` + WorkflowName string `bson:"workflow_name" json:"workflow_name"` + TaskID int64 `bson:"task_id" json:"task_id"` + FileName string `bson:"file_name" json:"file_name"` } //type Item struct { @@ -112,9 +130,11 @@ type DockerBuildStatus struct { } type JobCtx struct { - EnableProxy bool `bson:"enable_proxy" json:"enable_proxy"` - Proxy *models.Proxy `bson:"proxy" json:"proxy"` - CleanWorkspace bool `bson:"clean_workspace" json:"clean_workspace"` + EnableProxy bool `bson:"enable_proxy" json:"enable_proxy"` + Proxy *models.Proxy `bson:"proxy" json:"proxy"` + + // TODO: Deprecated. + CleanWorkspace bool `bson:"clean_workspace" json:"clean_workspace"` // BuildJobCtx Builds []*types.Repository `bson:"builds" json:"builds"` @@ -136,9 +156,9 @@ type JobCtx struct { DockerBuildCtx *DockerBuildCtx `bson:"docker_build_ctx,omitempty" json:"docker_build_ctx,omitempty"` FileArchiveCtx *FileArchiveCtx `bson:"file_archive_ctx,omitempty" json:"file_archive_ctx,omitempty"` // TestType - TestType string `bson:"test_type" json:"test_type"` - // Caches + TestType string `bson:"test_type" json:"test_type"` Caches []string `bson:"caches" json:"caches"` + ArtifactPath string `bson:"artifact_path,omitempty" json:"artifact_path,omitempty"` ArtifactPaths []string `bson:"artifact_paths,omitempty" json:"artifact_paths,omitempty"` IsHasArtifact bool `bson:"is_has_artifact" json:"is_has_artifact"` // StorageUri is used for qbox release-candidates @@ -148,6 +168,11 @@ type JobCtx struct { ClassicBuild bool `bson:"classic_build" json:"classic_build"` PostScripts string `bson:"post_scripts,omitempty" json:"post_scripts"` PMDeployScripts string `bson:"pm_deploy_scripts,omitempty" json:"pm_deploy_scripts"` + + // Upload To S3 related context + UploadEnabled bool `bson:"upload_enabled" json:"upload_enabled"` + UploadStorageInfo *types.ObjectStorageInfo `bson:"upload_storage_info" json:"upload_storage_info"` + UploadInfo []*types.ObjectStoragePathDetail `bson:"upload_info" json:"upload_info"` } type BuildStep struct { @@ -161,6 +186,7 @@ type SSH struct { Name string `json:"name"` UserName string `json:"user_name"` IP string `json:"ip"` + Port int64 `json:"port"` IsProd bool `json:"is_prod"` Label string `json:"label"` PrivateKey string `json:"private_key"` diff --git a/pkg/microservice/aslan/core/common/repository/models/task/deploy.go b/pkg/microservice/aslan/core/common/repository/models/task/deploy.go index 7ac5c6c2d4bce043061a55bfd39af4a0b31e7bf4..62f24c1c71985c3a0a7607d614989b8ffb28749f 100644 --- a/pkg/microservice/aslan/core/common/repository/models/task/deploy.go +++ b/pkg/microservice/aslan/core/common/repository/models/task/deploy.go @@ -20,6 +20,7 @@ import ( "fmt" "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/setting" ) type Resource struct { @@ -31,26 +32,28 @@ type Resource struct { // Deploy 容器部署任务 type Deploy struct { - TaskType config.TaskType `bson:"type" json:"type"` - Enabled bool `bson:"enabled" json:"enabled"` - TaskStatus config.Status `bson:"status" json:"status"` - Namespace string `bson:"namespace" json:"namespace"` - EnvName string `bson:"env_name" json:"env_name"` - ProductName string `bson:"product_name" json:"product_name"` - ServiceName string `bson:"service_name" json:"service_name"` - ServiceType string `bson:"service_type" json:"service_type"` - ServiceRevision int64 `bson:"service_revision,omitempty" json:"service_revision,omitempty"` - ContainerName string `bson:"container_name" json:"container_name"` - Image string `bson:"image" json:"image"` - Timeout int `bson:"timeout,omitempty" json:"timeout,omitempty"` - Error string `bson:"error,omitempty" json:"error,omitempty"` - StartTime int64 `bson:"start_time,omitempty" json:"start_time,omitempty"` - EndTime int64 `bson:"end_time,omitempty" json:"end_time,omitempty"` - ClusterID string `bson:"cluster_id,omitempty" json:"cluster_id,omitempty"` - ReplaceResources []Resource `bson:"replace_resources" json:"replace_resources"` - SkipWaiting bool `bson:"skipWaiting" json:"skipWaiting"` - IsRestart bool `bson:"is_restart" json:"is_restart"` - ResetImage bool `bson:"reset_image" json:"reset_image"` + TaskType config.TaskType `bson:"type" json:"type"` + Enabled bool `bson:"enabled" json:"enabled"` + TaskStatus config.Status `bson:"status" json:"status"` + Namespace string `bson:"namespace" json:"namespace"` + EnvName string `bson:"env_name" json:"env_name"` + ProductName string `bson:"product_name" json:"product_name"` + ServiceName string `bson:"service_name" json:"service_name"` + ServiceType string `bson:"service_type" json:"service_type"` + ServiceRevision int64 `bson:"service_revision,omitempty" json:"service_revision,omitempty"` + ReleaseName string `bson:"release_name" json:"release_name"` + ContainerName string `bson:"container_name" json:"container_name"` + Image string `bson:"image" json:"image"` + Timeout int `bson:"timeout,omitempty" json:"timeout,omitempty"` + Error string `bson:"error,omitempty" json:"error,omitempty"` + StartTime int64 `bson:"start_time,omitempty" json:"start_time,omitempty"` + EndTime int64 `bson:"end_time,omitempty" json:"end_time,omitempty"` + ClusterID string `bson:"cluster_id,omitempty" json:"cluster_id,omitempty"` + ReplaceResources []Resource `bson:"replace_resources" json:"replace_resources"` + SkipWaiting bool `bson:"skipWaiting" json:"skipWaiting"` + IsRestart bool `bson:"is_restart" json:"is_restart"` + ResetImage bool `bson:"reset_image" json:"reset_image"` + ResetImagePolicy setting.ResetImagePolicyType `bson:"reset_image_policy" json:"reset_image_policy"` } // SetNamespace ... diff --git a/pkg/microservice/aslan/core/common/repository/models/task/docker_build.go b/pkg/microservice/aslan/core/common/repository/models/task/docker_build.go index fe6162731d84a7d54e445579fda9bad120d288d0..387ec8cf28506d60d9ade13090b24581a83fa79e 100644 --- a/pkg/microservice/aslan/core/common/repository/models/task/docker_build.go +++ b/pkg/microservice/aslan/core/common/repository/models/task/docker_build.go @@ -54,11 +54,13 @@ func (db *DockerBuild) ToSubTask() (map[string]interface{}, error) { // DockerFile: dockerfile名称, 默认为Dockerfile // ImageBuild: build image镜像全称, e.g. xxx.com/release-candidates/image:tag type DockerBuildCtx struct { - WorkDir string `yaml:"work_dir" bson:"work_dir" json:"work_dir"` - DockerFile string `yaml:"docker_file" bson:"docker_file" json:"docker_file"` - ImageName string `yaml:"image_name" bson:"image_name" json:"image_name"` - BuildArgs string `yaml:"build_args" bson:"build_args" json:"build_args"` - ImageReleaseTag string `yaml:"image_release_tag,omitempty" bson:"image_release_tag,omitempty" json:"image_release_tag"` + Source string `yaml:"source" bson:"source" json:"source"` + WorkDir string `yaml:"work_dir" bson:"work_dir" json:"work_dir"` + DockerFile string `yaml:"docker_file" bson:"docker_file" json:"docker_file"` + ImageName string `yaml:"image_name" bson:"image_name" json:"image_name"` + BuildArgs string `yaml:"build_args" bson:"build_args" json:"build_args"` + ImageReleaseTag string `yaml:"image_release_tag,omitempty" bson:"image_release_tag,omitempty" json:"image_release_tag"` + DockerTemplateContent string `yaml:"docker_template_content" bson:"docker_template_content" json:"docker_template_content"` } type FileArchiveCtx struct { diff --git a/pkg/microservice/aslan/core/common/repository/models/task/extension.go b/pkg/microservice/aslan/core/common/repository/models/task/extension.go new file mode 100644 index 0000000000000000000000000000000000000000..97e3ee9a1d636aee6039dfd61ade149bc0c7977f --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/models/task/extension.go @@ -0,0 +1,57 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package task + +import ( + "fmt" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" +) + +type Extension struct { + TaskType config.TaskType `bson:"type" json:"type"` + Enabled bool `bson:"enabled" json:"enabled"` + TaskStatus config.Status `bson:"status" json:"status"` + URL string `bson:"url" json:"url"` + Path string `bson:"path,omitempty" json:"path,omitempty"` + IsCallback bool `bson:"is_callback" json:"is_callback"` + Headers []*models.KeyVal `bson:"headers,omitempty" json:"headers"` + Payload string `bson:"payload" json:"payload"` + ResponseBody *string `bson:"response_body" json:"response_body"` + ResponseCode *int `bson:"response_code" json:"response_code"` + ServiceInfos []*ServiceInfo `bson:"service_infos" json:"service_infos"` + Timeout int `bson:"timeout" json:"timeout,omitempty"` + IsRestart bool `bson:"is_restart" json:"is_restart"` + Error string `bson:"error,omitempty" json:"error,omitempty"` + StartTime int64 `bson:"start_time" json:"start_time,omitempty"` + EndTime int64 `bson:"end_time" json:"end_time,omitempty"` +} + +type ServiceInfo struct { + ServiceName string `bson:"service_name" json:"service_name"` + ServiceModule string `bson:"service_module" json:"service_module"` + Image string `bson:"image" json:"image"` +} + +func (t *Extension) ToSubTask() (map[string]interface{}, error) { + var task map[string]interface{} + if err := IToi(t, &task); err != nil { + return nil, fmt.Errorf("convert extension to interface error: %s", err) + } + return task, nil +} diff --git a/pkg/microservice/aslan/core/common/repository/models/task/jenkins.go b/pkg/microservice/aslan/core/common/repository/models/task/jenkins.go index 43f8946b43658e43bc127bacbb182924a3ac60db..a329a3d524992532e7ee133b8aed529fbd116466 100644 --- a/pkg/microservice/aslan/core/common/repository/models/task/jenkins.go +++ b/pkg/microservice/aslan/core/common/repository/models/task/jenkins.go @@ -20,7 +20,9 @@ import ( "fmt" "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/types" ) type JenkinsIntegration struct { @@ -30,33 +32,30 @@ type JenkinsIntegration struct { } type JenkinsBuildArgs struct { - JobName string `bson:"job_name" json:"job_name"` - JenkinsBuildParams []*JenkinsBuildParam `bson:"jenkins_build_param" json:"jenkins_build_params"` -} - -type JenkinsBuildParam struct { - Name string `json:"name"` - Value interface{} `json:"value"` + JobName string `bson:"job_name" json:"job_name"` + JenkinsBuildParams []*types.JenkinsBuildParam `bson:"jenkins_build_param" json:"jenkins_build_params"` } // JenkinsBuild ... type JenkinsBuild struct { - TaskType config.TaskType `bson:"type" json:"type"` - Enabled bool `bson:"enabled" json:"enabled"` - TaskStatus config.Status `bson:"status" json:"status"` - ServiceName string `bson:"service_name" json:"service_name"` - Service string `bson:"service" json:"service"` - OnSetup string `bson:"setup,omitempty" json:"setup,omitempty"` - Timeout int `bson:"timeout" json:"timeout,omitempty"` - Error string `bson:"error" json:"error,omitempty"` - ResReq setting.Request `bson:"res_req" json:"res_req"` - JenkinsIntegration *JenkinsIntegration `bson:"-" json:"jenkins_integration"` - JenkinsBuildArgs *JenkinsBuildArgs `bson:"-" json:"jenkins_build_args"` - StartTime int64 `bson:"start_time" json:"start_time,omitempty"` - EndTime int64 `bson:"end_time" json:"end_time,omitempty"` - LogFile string `bson:"log_file" json:"log_file"` - Image string `bson:"image,omitempty" json:"image,omitempty"` - IsRestart bool `bson:"is_restart" json:"is_restart"` + TaskType config.TaskType `bson:"type" json:"type"` + Enabled bool `bson:"enabled" json:"enabled"` + TaskStatus config.Status `bson:"status" json:"status"` + ServiceName string `bson:"service_name" json:"service_name"` + Service string `bson:"service" json:"service"` + OnSetup string `bson:"setup,omitempty" json:"setup,omitempty"` + Timeout int `bson:"timeout,omitempty" json:"timeout,omitempty"` + Error string `bson:"error,omitempty" json:"error,omitempty"` + ResReq setting.Request `bson:"res_req" json:"res_req"` + ResReqSpec setting.RequestSpec `bson:"res_req_spec" json:"res_req_spec"` + JenkinsIntegration *JenkinsIntegration `bson:"-" json:"jenkins_integration"` + JenkinsBuildArgs *JenkinsBuildArgs `bson:"-" json:"jenkins_build_args"` + Registries []*models.RegistryNamespace `bson:"registries" json:"registries"` + StartTime int64 `bson:"start_time,omitempty" json:"start_time,omitempty"` + EndTime int64 `bson:"end_time,omitempty" json:"end_time,omitempty"` + LogFile string `bson:"log_file" json:"log_file"` + Image string `bson:"image,omitempty" json:"image,omitempty"` + IsRestart bool `bson:"is_restart" json:"is_restart"` } // ToSubTask ... diff --git a/pkg/microservice/aslan/core/common/repository/models/task/model.go b/pkg/microservice/aslan/core/common/repository/models/task/model.go index b71dabdb5f82412a01510019c73f01a2df4fd14a..3131ba480fede8384d3b91cfc0767d9465a99e50 100644 --- a/pkg/microservice/aslan/core/common/repository/models/task/model.go +++ b/pkg/microservice/aslan/core/common/repository/models/task/model.go @@ -25,10 +25,11 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/setting" ) type Task struct { - ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` TaskID int64 `bson:"task_id" json:"task_id"` ProductName string `bson:"product_name" json:"product_name"` PipelineName string `bson:"pipeline_name" json:"pipeline_name"` @@ -48,7 +49,7 @@ type Task struct { TeamName string `bson:"team,omitempty" json:"team,omitempty"` IsDeleted bool `bson:"is_deleted" json:"is_deleted"` IsArchived bool `bson:"is_archived" json:"is_archived"` - AgentID string `bson:"agent_id" json:"agent_id"` + AgentID string `bson:"agent_id" json:"agent_id"` // 是否允许同时运行多次 MultiRun bool `bson:"multi_run" json:"multi_run"` // target 服务名称, k8s为容器名称, 物理机为服务名 @@ -57,45 +58,36 @@ type Task struct { // 查询条件为 服务模板名称: ServiceTmpl, 版本: BuildModuleVer // 如果为空，则使用pipeline自定义SubTasks BuildModuleVer string `bson:"build_module_ver,omitempty" json:"build_module_ver"` - - ServiceName string `bson:"service_name,omitempty" json:"service_name,omitempty"` + ServiceName string `bson:"service_name,omitempty" json:"service_name,omitempty"` // TaskArgs 单服务工作流任务参数 TaskArgs *models.TaskArgs `bson:"task_args,omitempty" json:"task_args,omitempty"` // WorkflowArgs 多服务工作流任务参数 WorkflowArgs *models.WorkflowTaskArgs `bson:"workflow_args" json:"workflow_args,omitempty"` // TestArgs 测试任务参数 TestArgs *models.TestTaskArgs `bson:"test_args,omitempty" json:"test_args,omitempty"` + // ScanningArgs argument for scanning tasks + ScanningArgs *models.ScanningArgs `bson:"scanning_args,omitempty" json:"scanning_args,omitempty"` // ServiceTaskArgs 脚本部署工作流任务参数 ServiceTaskArgs *models.ServiceTaskArgs `bson:"service_args,omitempty" json:"service_args,omitempty"` + // ArtifactPackageTaskArgs arguments for artifact-package type tasks + ArtifactPackageTaskArgs *models.ArtifactPackageTaskArgs `bson:"artifact_package_args,omitempty" json:"artifact_package_args,omitempty"` // ConfigPayload 系统配置信息 - ConfigPayload *models.ConfigPayload `json:"config_payload,omitempty"` - Error string `bson:"error,omitempty" json:"error,omitempty"` - // OrgID 单租户ID - OrgID int `bson:"org_id,omitempty" json:"org_id,omitempty"` - Services [][]*models.ProductService `bson:"services" json:"services"` - Render *models.RenderInfo `bson:"render" json:"render"` - StorageURI string `bson:"storage_uri,omitempty" json:"storage_uri,omitempty"` + ConfigPayload *models.ConfigPayload `bson:"configpayload" json:"config_payload,omitempty"` + Error string `bson:"error,omitempty" json:"error,omitempty"` + Services [][]*models.ProductService `bson:"services" json:"services"` + Render *models.RenderInfo `bson:"render" json:"render"` + StorageURI string `bson:"storage_uri,omitempty" json:"storage_uri,omitempty"` // interface{} 为types.TestReport - TestReports map[string]interface{} `bson:"test_reports,omitempty" json:"test_reports,omitempty"` - - RwLock sync.Mutex `bson:"-" json:"-"` - - ResetImage bool `json:"resetImage" bson:"resetImage"` - - TriggerBy *models.TriggerBy `json:"trigger_by,omitempty" bson:"trigger_by,omitempty"` - - Features []string `bson:"features" json:"features"` - IsRestart bool `bson:"is_restart" json:"is_restart"` - StorageEndpoint string `bson:"storage_endpoint" json:"storage_endpoint"` + TestReports map[string]interface{} `bson:"test_reports,omitempty" json:"test_reports,omitempty"` + RwLock sync.Mutex `bson:"-" json:"-"` + ResetImage bool `bson:"resetImage" json:"resetImage"` + ResetImagePolicy setting.ResetImagePolicyType `bson:"reset_image_policy" json:"reset_image_policy"` + TriggerBy *models.TriggerBy `bson:"trigger_by,omitempty" json:"trigger_by,omitempty"` + Features []string `bson:"features" json:"features"` + IsRestart bool `bson:"is_restart" json:"is_restart"` + StorageEndpoint string `bson:"storage_endpoint" json:"storage_endpoint"` } -//type RenderInfo struct { -// Name string `bson:"name" json:"name"` -// Revision int64 `bson:"revision" json:"revision"` -// ProductTmpl string `bson:"product_tmpl" json:"product_tmpl"` -// Description string `bson:"description" json:"description"` -//} - func (Task) TableName() string { return "pipeline_task_v2" } @@ -112,3 +104,73 @@ func IToi(before interface{}, after interface{}) error { return nil } + +type TestReport struct { + FunctionTestSuite *TestSuite `bson:"function_test_suite,omitempty" json:"functionTestSuite,omitempty"` + PerformanceTestSuites []*PerformanceTestSuite `bson:"performance_test_suite,omitempty" json:"performanceTestSuite,omitempty"` + Security map[string]map[string]int `bson:"security,omitempty" json:"security,omitempty"` +} + +type TestSuite struct { + // totalNum=tests+skips successNum=tests-failures-errors + Tests int `bson:"tests" json:"tests" xml:"tests,attr"` + Failures int `bson:"failures" json:"failures" xml:"failures,attr"` + Successes int `bson:"successes,omitempty" json:"successes,omitempty" xml:"successes,attr,omitempty"` + Skips int `bson:"skips" json:"skips" xml:"skips,attr"` + Errors int `bson:"errors,omitempty" json:"errors" xml:"errors,attr,omitempty"` + Time float64 `bson:"time" json:"time" xml:"time,attr"` + SystemOut string `bson:"system_out,omitempty" json:"system_out" xml:"system-out,omitempty"` + SystemErr string `bson:"system-err,omitempty" json:"system_err" xml:"system-err,omitempty"` + TestCases []TestCase `bson:"testcase" json:"testcase" xml:"testcase"` + SuiteType string `bson:"-" json:"-" xml:"-"` + Name string `bson:"name" json:"-" xml:"-"` +} + +type PerformanceTestSuite struct { + Label string `bson:"label" json:"label"` + Samples string `bson:"samples" json:"samples"` + Average string `bson:"average" json:"average"` + Min string `bson:"min" json:"min"` + Max string `bson:"max" json:"max"` + Line string `bson:"line" json:"line"` + StdDev string `bson:"std_dev" json:"stdDev"` + Error string `bson:"error" json:"error"` + Throughput string `bson:"throughput" json:"throughput"` + ReceivedKb string `bson:"received_kb" json:"receivedKb"` + AvgByte string `bson:"avg_byte" json:"avgByte"` +} + +type Skipped struct { +} + +type Failure struct { + Message string `bson:"message" json:"message" xml:"message,attr"` + Type string `bson:"type" json:"type" xml:"type,attr"` + Text string `bson:"text" json:"text" xml:",chardata"` +} + +type TestCase struct { + Name string `bson:"tc_name" json:"tc_name" xml:"name,attr"` + ClassName string `bson:"classname" json:"classname" xml:"classname,attr"` + Time float64 `bson:"time" json:"time" xml:"time,attr"` + Failure *Failure `bson:"failure,omitempty" json:"failure" xml:"failure,omitempty"` + Skipped *Skipped `bson:"skipped,omitempty" json:"skipped" xml:"skipped,omitempty"` + SystemOut string `bson:"system_out,omitempty" json:"system_out" xml:"system-out,omitempty"` + SystemErr string `bson:"system-err,omitempty" json:"system_err" xml:"system-err,omitempty"` + Error *Error `bson:"error,omitempty" json:"error" xml:"error,omitempty"` +} + +type Error struct { + Message string `bson:"message" json:"message" xml:"message,attr"` + Type string `bson:"type" json:"type" xml:"type,attr"` + Text string `bson:"text" json:"text" xml:",chardata"` +} + +type TaskOpt struct { + Task *Task `json:"task"` + EnvName string `json:"env_name"` + ServiceName string `json:"service_name"` + ServiceInfos *[]*ServiceInfo `json:"service_infos"` + IsWorkflowTask bool `json:"is_workflow_task"` + ImageName string `json:"image_name"` +} diff --git a/pkg/microservice/aslan/core/common/repository/models/task/release_image.go b/pkg/microservice/aslan/core/common/repository/models/task/release_image.go index c697c08b6bd2c6f2599b5edbc40ec1137a1ea921..3dd921ad95a0a4b0539c218377eb346dbd508af9 100644 --- a/pkg/microservice/aslan/core/common/repository/models/task/release_image.go +++ b/pkg/microservice/aslan/core/common/repository/models/task/release_image.go @@ -23,15 +23,6 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" ) -//type RepoImage struct { -// RepoID string `json:"repo_id" bson:"repo_id"` -// Name string `json:"name" bson:"name" yaml:"name"` -// Username string `json:"-" yaml:"username"` -// Password string `json:"-" yaml:"password"` -// Host string `json:"host" yaml:"host"` -// Namespace string `json:"namespace" yaml:"namespace"` -//} - type ReleaseImageItem struct { RepoID string `bson:"id" json:"id"` Image string `bson:"image" json:"image"` @@ -53,12 +44,37 @@ type ReleaseImage struct { // destinations to distribute images Releases []models.RepoImage `bson:"releases" json:"releases"` + + // New Version field + ProductName string `bson:"product_name" json:"product_name"` + SourceImage string `bson:"source_image" json:"source_image"` + DistributeInfo []*DistributeInfo `bson:"distribute_info" json:"distribute_info"` +} + +type DistributeInfo struct { + Image string `bson:"image" json:"image"` + ReleaseName string `bson:"release_name" json:"release_name"` + DistributeStartTime int64 `bson:"distribute_start_time" json:"distribute_start_time"` + DistributeEndTime int64 `bson:"distribute_end_time" json:"distribute_end_time"` + DistributeStatus string `bson:"distribute_status" json:"distribute_status"` + DeployEnabled bool `bson:"deploy_enabled" json:"deploy_enabled"` + DeployEnv string `bson:"deploy_env" json:"deploy_env"` + DeployServiceType string `json:"deploy_service_type" yaml:"deploy_service_type"` + DeployContainerName string `json:"deploy_container_name" yaml:"deploy_container_name"` + DeployServiceName string `json:"deploy_service_name" yaml:"deploy_service_name"` + DeployClusterID string `json:"deploy_cluster_id" yaml:"deploy_cluster_id"` + DeployNamespace string `bson:"deploy_namespace" json:"deploy_namespace"` + DeployStartTime int64 `bson:"deploy_start_time" json:"deploy_start_time"` + DeployEndTime int64 `bson:"deploy_end_time" json:"deploy_end_time"` + DeployStatus string `bson:"deploy_status" json:"deploy_status"` + RepoID string `bson:"repo_id" json:"repo_id"` } // SetImage ... func (ri *ReleaseImage) SetImage(image string) { if image != "" { ri.ImageTest = image + ri.SourceImage = image } } diff --git a/pkg/microservice/aslan/core/common/repository/models/task/scanning.go b/pkg/microservice/aslan/core/common/repository/models/task/scanning.go new file mode 100644 index 0000000000000000000000000000000000000000..f38c90f48a445857965b0ebf4633a39124a3e677 --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/models/task/scanning.go @@ -0,0 +1,56 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package task + +import ( + "fmt" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/types" +) + +type Scanning struct { + TaskType config.TaskType `bson:"type" json:"type"` + Status config.Status `bson:"status" json:"status,omitempty"` + ScanningID string `bson:"scanning_id" json:"scanning_id"` + Name string `bson:"name" json:"name"` + Error string `bson:"error,omitempty" json:"error,omitempty"` + ImageInfo string `bson:"image_info" json:"image_info"` + SonarInfo *types.SonarInfo `bson:"sonar_info" json:"sonar_info"` + Repos []*types.Repository `bson:"repos" json:"repos"` + Proxy *models.Proxy `bson:"proxy" json:"proxy"` + ClusterID string `bson:"cluster_id" json:"cluster_id"` + // ResReq defines job requested resources + ResReq setting.Request `bson:"res_req" json:"res_req"` + ResReqSpec setting.RequestSpec `bson:"res_req_spec" json:"res_req_spec"` + Timeout int64 `bson:"timeout" json:"timeout"` + Registries []*models.RegistryNamespace `bson:"-" json:"registries"` + // Parameter is for sonarQube type only + Parameter string `bson:"parameter" json:"parameter"` + // Script is for other type only + Script string `bson:"script" json:"script"` +} + +func (t *Scanning) ToSubTask() (map[string]interface{}, error) { + var task map[string]interface{} + if err := models.IToi(t, &task); err != nil { + return nil, fmt.Errorf("convert ScanningTask to interface error: %s", err) + } + return task, nil +} diff --git a/pkg/microservice/aslan/core/common/repository/models/task/testing.go b/pkg/microservice/aslan/core/common/repository/models/task/testing.go index f6aaab8a35c68ad4ce10819e2b587a706be3fcfd..cca8438aabdd832049662854044da8f4312fe291 100644 --- a/pkg/microservice/aslan/core/common/repository/models/task/testing.go +++ b/pkg/microservice/aslan/core/common/repository/models/task/testing.go @@ -21,6 +21,8 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/types" ) type Testing struct { @@ -39,12 +41,21 @@ type Testing struct { ImageFrom string `bson:"image_from" json:"image_from,omitempty"` ImageID string `bson:"image_id" json:"image_id"` // ResReq defines job requested resources - ResReq config.Request `bson:"res_req" json:"res_req"` + ResReq setting.Request `bson:"res_req" json:"res_req"` + ResReqSpec setting.RequestSpec `bson:"res_req_spec" json:"res_req_spec"` LogFile string `bson:"log_file" json:"log_file"` TestModuleName string `bson:"test_module_name" json:"test_module_name"` ReportReady bool `bson:"report_ready" json:"report_ready"` IsRestart bool `bson:"is_restart" json:"is_restart"` Registries []*models.RegistryNamespace `bson:"-" json:"registries"` + ClusterID string `bson:"cluster_id,omitempty" json:"cluster_id,omitempty"` + Namespace string `bson:"namespace" json:"namespace"` + + // New since V1.10.0. + Cache types.Cache `bson:"cache" json:"cache"` + CacheEnable bool `bson:"cache_enable" json:"cache_enable"` + CacheDirType types.CacheDirType `bson:"cache_dir_type" json:"cache_dir_type"` + CacheUserDir string `bson:"cache_user_dir" json:"cache_user_dir"` } func (t *Testing) ToSubTask() (map[string]interface{}, error) { diff --git a/pkg/microservice/aslan/core/common/repository/models/task/trigger.go b/pkg/microservice/aslan/core/common/repository/models/task/trigger.go new file mode 100644 index 0000000000000000000000000000000000000000..1487b8fcfe3ab9296d2b0bc60d2c9def17ddaeb7 --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/models/task/trigger.go @@ -0,0 +1,54 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package task + +import ( + "fmt" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" +) + +type Trigger struct { + TaskType config.TaskType `bson:"type" json:"type"` + Enabled bool `bson:"enabled" json:"enabled"` + TaskStatus config.Status `bson:"status" json:"status"` + URL string `bson:"url,omitempty" json:"url,omitempty"` + Path string `bson:"path,omitempty" json:"path,omitempty"` + IsCallback bool `bson:"is_callback,omitempty" json:"is_callback,omitempty"` + CallbackType string `bson:"callback_type,omitempty" json:"callback_type,omitempty"` + CallbackPayload *CallbackPayload `bson:"callback_payload,omitempty" json:"callback_payload,omitempty"` + Headers []*models.KeyVal `bson:"headers,omitempty" json:"headers,omitempty"` + Timeout int `bson:"timeout" json:"timeout,omitempty"` + IsRestart bool `bson:"is_restart" json:"is_restart"` + Error string `bson:"error,omitempty" json:"error,omitempty"` + StartTime int64 `bson:"start_time" json:"start_time,omitempty"` + EndTime int64 `bson:"end_time" json:"end_time,omitempty"` +} + +type CallbackPayload struct { + QRCodeURL string `bson:"QR_code_URL,omitempty" json:"QR_code_URL,omitempty"` +} + +// ToSubTask ... +func (t *Trigger) ToSubTask() (map[string]interface{}, error) { + var task map[string]interface{} + if err := IToi(t, &task); err != nil { + return nil, fmt.Errorf("convert trigger to interface error: %s", err) + } + return task, nil +} diff --git a/pkg/microservice/aslan/core/common/repository/models/template/product.go b/pkg/microservice/aslan/core/common/repository/models/template/product.go index 97f772e1a083eec1058f38477f225de7d4219867..fbd7d0479dcb09794754b7153ea4a2665f12a459 100644 --- a/pkg/microservice/aslan/core/common/repository/models/template/product.go +++ b/pkg/microservice/aslan/core/common/repository/models/template/product.go @@ -29,49 +29,48 @@ type Product struct { ProductName string `bson:"product_name" json:"product_name"` Revision int64 `bson:"revision" json:"revision"` CreateTime int64 `bson:"create_time" json:"create_time"` - Teams []*Team `bson:"teams" json:"teams"` - Team string `bson:"team" json:"team"` UpdateTime int64 `bson:"update_time" json:"update_time"` UpdateBy string `bson:"update_by" json:"update_by"` Enabled bool `bson:"enabled" json:"enabled"` Visibility string `bson:"visibility" json:"visibility"` + AutoDeploy *AutoDeployPolicy `bson:"auto_deploy" json:"auto_deploy"` Timeout int `bson:"timeout,omitempty" json:"timeout,omitempty"` Services [][]string `bson:"services" json:"services"` SharedServices []*ServiceInfo `bson:"shared_services,omitempty" json:"shared_services,omitempty"` Vars []*RenderKV `bson:"vars" json:"vars"` EnvVars []*EnvRenderKV `bson:"-" json:"env_vars,omitempty"` ChartInfos []*RenderChart `bson:"-" json:"chart_infos,omitempty"` - UserIDs []int `bson:"user_ids" json:"user_ids"` - TeamID int `bson:"team_id" json:"team_id"` Description string `bson:"description,omitempty" json:"desc,omitempty"` ProductFeature *ProductFeature `bson:"product_feature,omitempty" json:"product_feature,omitempty"` ImageSearchingRules []*ImageSearchingRule `bson:"image_searching_rules,omitempty" json:"image_searching_rules,omitempty"` // onboarding状态，0表示onboarding完成，1、2、3、4代表当前onboarding所在的步骤 OnboardingStatus int `bson:"onboarding_status" json:"onboarding_status"` // CI场景的onboarding流程创建的ci工作流id，用于前端跳转 - CiPipelineID string `bson:"-" json:"ci_pipeline_id"` - Role string `bson:"-" json:"role,omitempty"` - PermissionUUIDs []string `bson:"-" json:"permissionUUIDs"` - TotalServiceNum int `bson:"-" json:"total_service_num"` - LatestServiceUpdateTime int64 `bson:"-" json:"latest_service_update_time"` - LatestServiceUpdateBy string `bson:"-" json:"latest_service_update_by"` - TotalBuildNum int `bson:"-" json:"total_build_num"` - LatestBuildUpdateTime int64 `bson:"-" json:"latest_build_update_time"` - LatestBuildUpdateBy string `bson:"-" json:"latest_build_update_by"` - TotalTestNum int `bson:"-" json:"total_test_num"` - LatestTestUpdateTime int64 `bson:"-" json:"latest_test_update_time"` - LatestTestUpdateBy string `bson:"-" json:"latest_test_update_by"` - TotalEnvNum int `bson:"-" json:"total_env_num"` - LatestEnvUpdateTime int64 `bson:"-" json:"latest_env_update_time"` - LatestEnvUpdateBy string `bson:"-" json:"latest_env_update_by"` - TotalWorkflowNum int `bson:"-" json:"total_workflow_num"` - LatestWorkflowUpdateTime int64 `bson:"-" json:"latest_workflow_update_time"` - LatestWorkflowUpdateBy string `bson:"-" json:"latest_workflow_update_by"` - TotalEnvTemplateServiceNum int `bson:"-" json:"total_env_template_service_num"` - ShowProject bool `bson:"-" json:"show_project"` - IsOpensource bool `bson:"is_opensource" json:"is_opensource"` - CustomImageRule *CustomRule `bson:"custom_image_rule,omitempty" json:"custom_image_rule,omitempty"` - CustomTarRule *CustomRule `bson:"custom_tar_rule,omitempty" json:"custom_tar_rule,omitempty"` + CiPipelineID string `bson:"-" json:"ci_pipeline_id"` + Role string `bson:"-" json:"role,omitempty"` + PermissionUUIDs []string `bson:"-" json:"permissionUUIDs"` + TotalServiceNum int `bson:"-" json:"total_service_num"` + LatestServiceUpdateTime int64 `bson:"-" json:"latest_service_update_time"` + LatestServiceUpdateBy string `bson:"-" json:"latest_service_update_by"` + TotalBuildNum int `bson:"-" json:"total_build_num"` + LatestBuildUpdateTime int64 `bson:"-" json:"latest_build_update_time"` + LatestBuildUpdateBy string `bson:"-" json:"latest_build_update_by"` + TotalTestNum int `bson:"-" json:"total_test_num"` + LatestTestUpdateTime int64 `bson:"-" json:"latest_test_update_time"` + LatestTestUpdateBy string `bson:"-" json:"latest_test_update_by"` + TotalEnvNum int `bson:"-" json:"total_env_num"` + LatestEnvUpdateTime int64 `bson:"-" json:"latest_env_update_time"` + LatestEnvUpdateBy string `bson:"-" json:"latest_env_update_by"` + TotalWorkflowNum int `bson:"-" json:"total_workflow_num"` + LatestWorkflowUpdateTime int64 `bson:"-" json:"latest_workflow_update_time"` + LatestWorkflowUpdateBy string `bson:"-" json:"latest_workflow_update_by"` + TotalEnvTemplateServiceNum int `bson:"-" json:"total_env_template_service_num"` + ClusterIDs []string `bson:"-" json:"cluster_ids"` + IsOpensource bool `bson:"is_opensource" json:"is_opensource"` + CustomImageRule *CustomRule `bson:"custom_image_rule,omitempty" json:"custom_image_rule,omitempty"` + CustomTarRule *CustomRule `bson:"custom_tar_rule,omitempty" json:"custom_tar_rule,omitempty"` + DeliveryVersionHook *DeliveryVersionHook `bson:"delivery_version_hook" json:"delivery_version_hook"` + Public bool `bson:"public,omitempty" json:"public"` } type ServiceInfo struct { @@ -98,17 +97,26 @@ type EnvRenderKV struct { } type GitRepoConfig struct { - CodehostID int `bson:"codehost_id,omitempty"` - Owner string `bson:"owner,omitempty"` - Repo string `bson:"repo,omitempty"` - Branch string `bson:"branch,omitempty"` + CodehostID int `bson:"codehost_id,omitempty" json:"codehost_id"` + Owner string `bson:"owner,omitempty" json:"owner"` + Repo string `bson:"repo,omitempty" json:"repo"` + Branch string `bson:"branch,omitempty" json:"branch"` + Namespace string `bson:"namespace,omitempty" json:"namespace"` // records the actual namespace of repo, used to generate correct project name + ValuesPaths []string `bson:"values_paths,omitempty" json:"values_paths,omitempty"` +} + +func (grc *GitRepoConfig) GetNamespace() string { + if len(grc.Namespace) > 0 { + return grc.Namespace + } + return grc.Owner } type CustomYaml struct { - YamlSource string `bson:"yaml_source,omitempty" json:"yaml_source,omitempty"` - YamlContent string `bson:"yaml_content,omitempty" json:"yaml_content,omitempty"` - GitRepoConfig *GitRepoConfig `bson:"git_repo_config,omitempty" json:"git_repo_config,omitempty"` - ValuesPaths []string `bson:"values_paths,omitempty" json:"values_paths,omitempty"` + YamlContent string `bson:"yaml_content,omitempty" json:"yaml_content,omitempty"` + Source string `bson:"source" json:"source"` + AutoSync bool `bson:"auto_sync" json:"auto_sync"` + SourceDetail interface{} `bson:"source_detail" json:"source_detail"` } // RenderChart ... @@ -155,6 +163,17 @@ type CustomRule struct { BranchRule string `bson:"branch_rule,omitempty" json:"branch_rule,omitempty"` PRAndBranchRule string `bson:"pr_and_branch_rule,omitempty" json:"pr_and_branch_rule,omitempty"` TagRule string `bson:"tag_rule,omitempty" json:"tag_rule,omitempty"` + JenkinsRule string `bson:"jenkins_rule,omitempty" json:"jenkins_rule,omitempty"` +} + +type DeliveryVersionHook struct { + Enable bool `bson:"enable" json:"enable"` + HookHost string `bson:"hook_host" json:"hook_host"` + Path string `bson:"path" json:"path"` +} + +type AutoDeployPolicy struct { + Enable bool `bson:"enable" json:"enable"` } func (Product) TableName() string { @@ -231,22 +250,6 @@ func (r *RenderKV) RemoveDupServices() { r.Services = result } -func (rc *RenderChart) DiffOverrideYaml(other *RenderChart) bool { - if rc.OverrideYaml == nil { - if other.OverrideYaml != nil { - return true - } - return false - } - if other.OverrideYaml == nil { - return true - } - if rc.OverrideYaml.YamlSource != other.OverrideYaml.YamlSource || rc.OverrideYaml.YamlContent != other.OverrideYaml.YamlContent { - return true - } - return false -} - func (rc *RenderChart) GetOverrideYaml() string { if rc.OverrideYaml == nil { return "" diff --git a/pkg/microservice/aslan/core/common/repository/models/testing.go b/pkg/microservice/aslan/core/common/repository/models/testing.go index 7ea1b46e84ed069ce31292f64c6ee8327db08f64..4403866630a28048d6410b6d214d03c381c99b02 100644 --- a/pkg/microservice/aslan/core/common/repository/models/testing.go +++ b/pkg/microservice/aslan/core/common/repository/models/testing.go @@ -19,7 +19,7 @@ package models import ( "go.mongodb.org/mongo-driver/bson/primitive" - "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/setting" "github.com/koderover/zadig/pkg/types" ) @@ -30,7 +30,7 @@ type Testing struct { Desc string `bson:"desc" json:"desc"` Timeout int `bson:"timeout" json:"timeout"` Team string `bson:"team" json:"team"` - Repos []*types.Repository `bson:"repos,omitempty" json:"repos"` + Repos []*types.Repository `bson:"repos" json:"repos"` PreTest *PreTest `bson:"pre_test" json:"pre_test"` Scripts string `bson:"scripts" json:"scripts"` UpdateTime int64 `bson:"update_time" json:"update_time"` @@ -38,19 +38,33 @@ type Testing struct { // Junit 测试报告 TestResultPath string `bson:"test_result_path" json:"test_result_path"` // html 测试报告 - TestReportPath string `bson:"test_report_path" json:"test_report_path"` - Threshold int `bson:"threshold" json:"threshold"` - TestType string `bson:"test_type" json:"test_type"` - Caches []string `bson:"caches" json:"caches"` - ArtifactPaths []string `bson:"artifact_paths,omitempty" json:"artifact_paths,omitempty"` - TestCaseNum int `bson:"-" json:"test_case_num,omitempty"` - ExecuteNum int `bson:"-" json:"execute_num,omitempty"` - PassRate float64 `bson:"-" json:"pass_rate,omitempty"` - AvgDuration float64 `bson:"-" json:"avg_duration,omitempty"` - Workflows []*Workflow `bson:"-" json:"workflows,omitempty"` - Schedules *ScheduleCtrl `bson:"schedules,omitempty" json:"schedules,omitempty"` - HookCtl *TestingHookCtrl `bson:"hook_ctl" json:"hook_ctl"` - ScheduleEnabled bool `bson:"schedule_enabled" json:"-"` + TestReportPath string `bson:"test_report_path" json:"test_report_path"` + Threshold int `bson:"threshold" json:"threshold"` + TestType string `bson:"test_type" json:"test_type"` + + // TODO: Deprecated. + Caches []string `bson:"caches" json:"caches"` + + ArtifactPaths []string `bson:"artifact_paths,omitempty" json:"artifact_paths,omitempty"` + TestCaseNum int `bson:"-" json:"test_case_num,omitempty"` + ExecuteNum int `bson:"-" json:"execute_num,omitempty"` + PassRate float64 `bson:"-" json:"pass_rate,omitempty"` + AvgDuration float64 `bson:"-" json:"avg_duration,omitempty"` + Workflows []*Workflow `bson:"-" json:"workflows,omitempty"` + Schedules *ScheduleCtrl `bson:"schedules,omitempty" json:"schedules,omitempty"` + HookCtl *TestingHookCtrl `bson:"hook_ctl" json:"hook_ctl"` + // TODO: Deprecated. + NotifyCtl *NotifyCtl `bson:"notify_ctl,omitempty" json:"notify_ctl,omitempty"` + // New since V1.12.0. + NotifyCtls []*NotifyCtl `bson:"notify_ctls" json:"notify_ctls"` + ScheduleEnabled bool `bson:"schedule_enabled" json:"-"` + + // New since V1.10.0. + CacheEnable bool `bson:"cache_enable" json:"cache_enable"` + CacheDirType types.CacheDirType `bson:"cache_dir_type" json:"cache_dir_type"` + CacheUserDir string `bson:"cache_user_dir" json:"cache_user_dir"` + // New since V1.10.0. Only to tell the webpage should the advanced settings be displayed + AdvancedSettingsModified bool `bson:"advanced_setting_modified" json:"advanced_setting_modified"` } type TestingHookCtrl struct { @@ -60,25 +74,32 @@ type TestingHookCtrl struct { type TestingHook struct { AutoCancel bool `bson:"auto_cancel" json:"auto_cancel"` - MainRepo MainHookRepo `bson:"main_repo" json:"main_repo"` + MainRepo *MainHookRepo `bson:"main_repo" json:"main_repo"` TestArgs *TestTaskArgs `bson:"test_args" json:"test_args"` } // PreTest prepares an environment for a job type PreTest struct { + // TODO: Deprecated. CleanWorkspace bool `bson:"clean_workspace" json:"clean_workspace"` + // BuildOS defines job image OS, it supports 12.04, 14.04, 16.04 BuildOS string `bson:"build_os" json:"build_os"` ImageFrom string `bson:"image_from" json:"image_from"` ImageID string `bson:"image_id" json:"image_id"` // ResReq defines job requested resources - ResReq config.Request `bson:"res_req" json:"res_req"` + ResReq setting.Request `bson:"res_req" json:"res_req"` + ResReqSpec setting.RequestSpec `bson:"res_req_spec" json:"res_req_spec"` // Installs defines apps to be installed for build Installs []*Item `bson:"installs,omitempty" json:"installs"` // Envs stores user defined env key val for build Envs []*KeyVal `bson:"envs,omitempty" json:"envs"` // EnableProxy - EnableProxy bool `bson:"enable_proxy" json:"enable_proxy"` + EnableProxy bool `bson:"enable_proxy" json:"enable_proxy"` + ClusterID string `bson:"cluster_id" json:"cluster_id"` + + // TODO: Deprecated. + Namespace string `bson:"namespace" json:"namespace"` } func (Testing) TableName() string { diff --git a/pkg/microservice/aslan/core/common/repository/models/wokflow_task_v4.go b/pkg/microservice/aslan/core/common/repository/models/wokflow_task_v4.go new file mode 100644 index 0000000000000000000000000000000000000000..0ba8de43a1960537fd716b0b56b414fe3041a3ab --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/models/wokflow_task_v4.go @@ -0,0 +1,99 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" +) + +type WorkflowTask struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + TaskID int64 `bson:"task_id" json:"task_id"` + WorkflowName string `bson:"workflow_name" json:"workflow_name"` + WorkflowArgs *WorkflowV4 `bson:"workflow_args" json:"workflow_args"` + KeyVals []*KeyVal `bson:"key_vals" json:"key_vals"` + GlobalContext map[string]string `bson:"global_context" json:"global_context"` + Status config.Status `bson:"status" json:"status,omitempty"` + TaskCreator string `bson:"task_creator" json:"task_creator,omitempty"` + TaskRevoker string `bson:"task_revoker,omitempty" json:"task_revoker,omitempty"` + CreateTime int64 `bson:"create_time" json:"create_time,omitempty"` + StartTime int64 `bson:"start_time" json:"start_time,omitempty"` + EndTime int64 `bson:"end_time" json:"end_time,omitempty"` + Stages []*StageTask `bson:"stages" json:"stages"` + ProjectName string `bson:"project_name,omitempty" json:"project_name,omitempty"` + IsDeleted bool `bson:"is_deleted" json:"is_deleted"` + IsArchived bool `bson:"is_archived" json:"is_archived"` + Error string `bson:"error,omitempty" json:"error,omitempty"` + IsRestart bool `bson:"is_restart" json:"is_restart"` + MultiRun bool `bson:"multi_run" json:"multi_run"` +} + +func (WorkflowTask) TableName() string { + return "workflow_task" +} + +type StageTask struct { + Name string `bson:"name" json:"name"` + Status config.Status `bson:"status" json:"status"` + StartTime int64 `bson:"start_time" json:"start_time,omitempty"` + EndTime int64 `bson:"end_time" json:"end_time,omitempty"` + Parallel bool `bson:"parallel" json:"parallel"` + Approval *Approval `bson:"approval" json:"approval"` + Jobs []*JobTask `bson:"jobs" json:"jobs"` + Error string `bson:"error" json:"error"` +} + +type JobTask struct { + Name string `bson:"name" json:"name"` + JobType string `bson:"type" json:"type"` + Status config.Status `bson:"status" json:"status"` + StartTime int64 `bson:"start_time" json:"start_time,omitempty"` + EndTime int64 `bson:"end_time" json:"end_time,omitempty"` + Error string `bson:"error" json:"error"` + Properties JobProperties `bson:"properties" json:"properties"` + Plugin *PluginTemplate `bson:"plugin" json:"plugin"` + Steps []*StepTask `bson:"steps" json:"steps"` + Outputs []*Output `bson:"outputs" json:"outputs"` +} + +type StepTask struct { + Name string `bson:"name" json:"name" yaml:"name"` + JobName string `bson:"job_name" json:"job_name" yaml:"job_name"` + Error string `bson:"error" json:"error" yaml:"error"` + StepType config.StepType `bson:"type" json:"type" yaml:"type"` + // step input params,differ form steps + Spec interface{} `bson:"spec" json:"spec" yaml:"spec"` + // step output results,like testing results,differ form steps + Result interface{} `bson:"result" json:"result" yaml:"result"` +} + +type WorkflowTaskCtx struct { + WorkflowName string + ProjectName string + TaskID int64 + DockerHost string + Workspace string + DistDir string + DockerMountDir string + ConfigMapMountDir string + WorkflowKeyVals []*KeyVal + GlobalContextGet func(key string) (string, bool) + GlobalContextSet func(key, value string) + GlobalContextEach func(f func(k, v string) bool) +} diff --git a/pkg/microservice/aslan/core/common/repository/models/workflow.go b/pkg/microservice/aslan/core/common/repository/models/workflow.go index 0eaa8dc823c5ee6aa6567821db6a53c2baa4ea40..bc8879d5824cd49c1b494fa46f54b17b5708de97 100644 --- a/pkg/microservice/aslan/core/common/repository/models/workflow.go +++ b/pkg/microservice/aslan/core/common/repository/models/workflow.go @@ -47,18 +47,17 @@ type Workflow struct { TestStage *TestStage `bson:"test_stage" json:"test_stage"` SecurityStage *SecurityStage `bson:"security_stage" json:"security_stage"` DistributeStage *DistributeStage `bson:"distribute_stage" json:"distribute_stage"` - NotifyCtl *NotifyCtl `bson:"notify_ctl,omitempty" json:"notify_ctl,omitempty"` - HookCtl *WorkflowHookCtrl `bson:"hook_ctl" json:"hook_ctl"` - IsFavorite bool `bson:"-" json:"is_favorite"` - LastestTask *TaskInfo `bson:"-" json:"lastest_task"` - LastSucessTask *TaskInfo `bson:"-" json:"last_task_success"` - LastFailureTask *TaskInfo `bson:"-" json:"last_task_failure"` - TotalDuration int64 `bson:"-" json:"total_duration"` - TotalNum int `bson:"-" json:"total_num"` - TotalSuccess int `bson:"-" json:"total_success"` + ExtensionStage *ExtensionStage `bson:"extension_stage" json:"extension_stage"` + // TODO: Deprecated. + NotifyCtl *NotifyCtl `bson:"notify_ctl,omitempty" json:"notify_ctl,omitempty"` + // New since V1.12.0. + NotifyCtls []*NotifyCtl `bson:"notify_ctls" json:"notify_ctls"` + HookCtl *WorkflowHookCtrl `bson:"hook_ctl" json:"hook_ctl"` + BaseName string `bson:"base_name" json:"base_name"` // ResetImage indicate whether reset image to original version after completion - ResetImage bool `json:"reset_image" bson:"reset_image"` + ResetImage bool `bson:"reset_image" json:"reset_image"` + ResetImagePolicy setting.ResetImagePolicyType `bson:"reset_image_policy,omitempty" json:"reset_image_policy,omitempty"` // IsParallel 控制单一工作流的任务是否支持并行处理 IsParallel bool `json:"is_parallel" bson:"is_parallel"` } @@ -69,24 +68,37 @@ type WorkflowHookCtrl struct { } type WorkflowHook struct { - AutoCancel bool `bson:"auto_cancel" json:"auto_cancel"` - CheckPatchSetChange bool `bson:"check_patch_set_change" json:"check_patch_set_change"` - MainRepo MainHookRepo `bson:"main_repo" json:"main_repo"` - WorkflowArgs *WorkflowTaskArgs `bson:"workflow_args" json:"workflow_args"` + AutoCancel bool `bson:"auto_cancel" json:"auto_cancel"` + CheckPatchSetChange bool `bson:"check_patch_set_change" json:"check_patch_set_change"` + MainRepo *MainHookRepo `bson:"main_repo" json:"main_repo"` + WorkflowArgs *WorkflowTaskArgs `bson:"workflow_args" json:"workflow_args"` + IsYaml bool `bson:"is_yaml,omitempty" json:"is_yaml,omitempty"` + YamlPath string `bson:"yaml_path,omitempty" json:"yaml_path,omitempty"` } type MainHookRepo struct { - Name string `bson:"name,omitempty" json:"name,omitempty"` - Description string `bson:"description,omitempty" json:"description,omitempty"` - Source string `bson:"source,omitempty" json:"source,omitempty"` - RepoOwner string `bson:"repo_owner" json:"repo_owner"` - RepoName string `bson:"repo_name" json:"repo_name"` - Branch string `bson:"branch" json:"branch"` - MatchFolders []string `bson:"match_folders" json:"match_folders,omitempty"` - CodehostID int `bson:"codehost_id" json:"codehost_id"` - Events []config.HookEventType `bson:"events" json:"events"` - Label string `bson:"label" json:"label"` - Revision string `bson:"revision" json:"revision"` + Name string `bson:"name,omitempty" json:"name,omitempty"` + Description string `bson:"description,omitempty" json:"description,omitempty"` + Source string `bson:"source,omitempty" json:"source,omitempty"` + RepoOwner string `bson:"repo_owner" json:"repo_owner"` + RepoNamespace string `bson:"repo_namespace" json:"repo_namespace"` + RepoName string `bson:"repo_name" json:"repo_name"` + Branch string `bson:"branch" json:"branch"` + Tag string `bson:"tag" json:"tag"` + Committer string `bson:"committer" json:"committer"` + MatchFolders []string `bson:"match_folders" json:"match_folders,omitempty"` + CodehostID int `bson:"codehost_id" json:"codehost_id"` + Events []config.HookEventType `bson:"events" json:"events"` + Label string `bson:"label" json:"label"` + Revision string `bson:"revision" json:"revision"` + IsRegular bool `bson:"is_regular" json:"is_regular"` +} + +func (m *MainHookRepo) GetRepoNamespace() string { + if m.RepoNamespace != "" { + return m.RepoNamespace + } + return m.RepoOwner } func (m MainHookRepo) GetLabelValue() string { @@ -134,24 +146,30 @@ type TaskArgs struct { CodeHostID int `bson:"codehost_id" json:"codehost_id"` } +type CallbackArgs struct { + CallbackUrl string `bson:"callback_url" json:"callback_url"` // url-encoded full path + CallbackVars map[string]interface{} `bson:"callback_vars" json:"callback_vars"` // custom defied vars, will be set to body of callback request +} + // WorkflowTaskArgs 多服务工作流任务参数 type WorkflowTaskArgs struct { WorkflowName string `bson:"workflow_name" json:"workflow_name"` ProductTmplName string `bson:"product_tmpl_name" json:"product_tmpl_name"` Description string `bson:"description,omitempty" json:"description,omitempty"` //为了兼容老数据，namespace可能会存多个环境名称，用逗号隔开 - Namespace string `bson:"namespace" json:"namespace"` - BaseNamespace string `bson:"base_namespace,omitempty" json:"base_namespace,omitempty"` - EnvRecyclePolicy string `bson:"env_recycle_policy,omitempty" json:"env_recycle_policy,omitempty"` - EnvUpdatePolicy string `bson:"env_update_policy,omitempty" json:"env_update_policy,omitempty"` - Target []*TargetArgs `bson:"targets" json:"targets"` - Artifact []*ArtifactArgs `bson:"artifact_args" json:"artifact_args"` - Tests []*TestArgs `bson:"tests" json:"tests"` - VersionArgs *VersionArgs `bson:"version_args,omitempty" json:"version_args,omitempty"` - ReqID string `bson:"req_id" json:"req_id"` - RegistryID string `bson:"registry_id,omitempty" json:"registry_id,omitempty"` - DistributeEnabled bool `bson:"distribute_enabled" json:"distribute_enabled"` - WorklowTaskCreator string `bson:"workflow_task_creator" json:"workflow_task_creator"` + Namespace string `bson:"namespace" json:"namespace"` + BaseNamespace string `bson:"base_namespace,omitempty" json:"base_namespace,omitempty"` + EnvRecyclePolicy string `bson:"env_recycle_policy,omitempty" json:"env_recycle_policy,omitempty"` + EnvUpdatePolicy string `bson:"env_update_policy,omitempty" json:"env_update_policy,omitempty"` + Target []*TargetArgs `bson:"targets" json:"targets"` + Artifact []*ArtifactArgs `bson:"artifact_args" json:"artifact_args"` + Tests []*TestArgs `bson:"tests" json:"tests"` + VersionArgs *VersionArgs `bson:"version_args,omitempty" json:"version_args,omitempty"` + ReqID string `bson:"req_id" json:"req_id"` + RegistryID string `bson:"registry_id,omitempty" json:"registry_id,omitempty"` + StorageID string `bson:"storage_id,omitempty" json:"storage_id,omitempty"` + DistributeEnabled bool `bson:"distribute_enabled" json:"distribute_enabled"` + WorkflowTaskCreator string `bson:"workflow_task_creator" json:"workflow_task_creator"` // Ignore docker build cache IgnoreCache bool `json:"ignore_cache" bson:"ignore_cache"` // Ignore workspace cache and reset volume @@ -166,13 +184,32 @@ type WorkflowTaskArgs struct { Source string `bson:"source" json:"source"` CodehostID int `bson:"codehost_id" json:"codehost_id"` RepoOwner string `bson:"repo_owner" json:"repo_owner"` + RepoNamespace string `bson:"repo_namespace" json:"repo_namespace"` RepoName string `bson:"repo_name" json:"repo_name"` - + Committer string `bson:"committer,omitempty" json:"committer,omitempty"` //github check run HookPayload *HookPayload `bson:"hook_payload" json:"hook_payload,omitempty"` // 请求模式，openAPI表示外部客户调用 RequestMode string `json:"request_mode,omitempty"` IsParallel bool `json:"is_parallel" bson:"is_parallel"` + EnvName string `json:"env_name" bson:"-"` + + Callback *CallbackArgs `bson:"callback" json:"callback"` + ReleaseImages []*ReleaseImage `bson:"release_images,omitempty" json:"release_images,omitempty"` +} + +type ScanningArgs struct { + ScanningName string `json:"scanning_name" bson:"scanning_name"` + ScanningID string `json:"scanning_id" bson:"scanning_id"` + + // NotificationID is the id of scmnotify.Notification + NotificationID string `bson:"notification_id" json:"notification_id"` +} + +type ReleaseImage struct { + Image string `bson:"image" json:"image"` + ServiceName string `bson:"service_name" json:"service_name"` + ServiceModule string `bson:"service_module" json:"service_module"` } type TestTaskArgs struct { @@ -187,6 +224,7 @@ type TestTaskArgs struct { Source string `bson:"source" json:"source"` CodehostID int `bson:"codehost_id" json:"codehost_id"` RepoOwner string `bson:"repo_owner" json:"repo_owner"` + RepoNamespace string `bson:"repo_namespace" json:"repo_namespace"` RepoName string `bson:"repo_name" json:"repo_name"` } @@ -204,8 +242,10 @@ type BuildStage struct { // BuildModule ... type BuildModule struct { - Target *ServiceModuleTarget `bson:"target" json:"target"` - BuildModuleVer string `bson:"build_module_ver" json:"build_module_ver"` + Target *ServiceModuleTarget `bson:"target" json:"target"` + HideServiceModule bool `bson:"hide_service_module" json:"hide_service_module"` + BuildModuleVer string `bson:"build_module_ver" json:"build_module_ver"` + BranchFilter []*types.BranchFilterInfo `bson:"branch_filter" json:"branch_filter"` } type ArtifactStage struct { @@ -215,7 +255,8 @@ type ArtifactStage struct { // ArtifactModule ... type ArtifactModule struct { - Target *ServiceModuleTarget `bson:"target" json:"target"` + HideServiceModule bool `bson:"hide_service_module" json:"hide_service_module"` + Target *ServiceModuleTarget `bson:"target" json:"target"` } type TestStage struct { @@ -245,13 +286,24 @@ type DistributeStage struct { Releases []RepoImage `bson:"releases" json:"releases"` } +type ExtensionStage struct { + Enabled bool `bson:"enabled" json:"enabled"` + URL string `bson:"url" json:"url"` + Path string `bson:"path" json:"path"` + IsCallback bool `bson:"is_callback" json:"is_callback"` + Timeout int `bson:"timeout" json:"timeout"` + Headers []*KeyVal `bson:"headers" json:"headers"` +} + type RepoImage struct { - RepoID string `json:"repo_id" bson:"repo_id"` - Name string `json:"name" bson:"name" yaml:"name"` - Username string `json:"-" yaml:"username"` - Password string `json:"-" yaml:"password"` - Host string `json:"host" yaml:"host"` - Namespace string `json:"namespace" yaml:"namespace"` + RepoID string `json:"repo_id" bson:"repo_id"` + Name string `json:"name" bson:"name" yaml:"name"` + Username string `json:"-" yaml:"username"` + Password string `json:"-" yaml:"password"` + Host string `json:"host" yaml:"host"` + Namespace string `json:"namespace" yaml:"namespace"` + DeployEnabled bool `json:"deploy_enabled" yaml:"deploy_enabled"` + DeployEnv string `json:"deploy_env" yaml:"deploy_env"` } type ProductDistribute struct { @@ -311,21 +363,24 @@ type HookPayload struct { } type TargetArgs struct { - Name string `bson:"name" json:"name"` - ServiceName string `bson:"service_name" json:"service_name"` - ProductName string `bson:"product_name" json:"product_name"` - Build *BuildArgs `bson:"build" json:"build"` - Deploy []DeployEnv `bson:"deloy" json:"deploy"` - Image string `bson:"image" json:"image"` - BinFile string `bson:"bin_file" json:"bin_file"` - Envs []*KeyVal `bson:"envs" json:"envs"` - HasBuild bool `bson:"has_build" json:"has_build"` - JenkinsBuildArgs *JenkinsBuildArgs `bson:"jenkins_build_args" json:"jenkins_build_args"` + Name string `bson:"name" json:"name"` + ImageName string `bson:"image_name" json:"image_name"` + ServiceName string `bson:"service_name" json:"service_name"` + ServiceType string `bson:"service_type,omitempty" json:"service_type,omitempty"` + ProductName string `bson:"product_name" json:"product_name"` + Build *BuildArgs `bson:"build" json:"build"` + Deploy []DeployEnv `bson:"deploy" json:"deploy"` + Image string `bson:"image,omitempty" json:"image,omitempty"` + BinFile string `bson:"bin_file" json:"bin_file"` + Envs []*KeyVal `bson:"envs" json:"envs"` + HasBuild bool `bson:"has_build" json:"has_build"` + JenkinsBuildArgs *JenkinsBuildArgs `bson:"jenkins_build_args,omitempty" json:"jenkins_build_args,omitempty"` + BuildName string `bson:"build_name,omitempty" json:"build_name"` } type JenkinsBuildArgs struct { - JobName string `bson:"job_name" json:"job_name"` - JenkinsBuildParams []*JenkinsBuildParam `bson:"jenkins_build_param" json:"jenkins_build_params"` + JobName string `bson:"job_name" json:"job_name"` + JenkinsBuildParams []*types.JenkinsBuildParam `bson:"jenkins_build_param" json:"jenkins_build_params"` } type BuildArgs struct { @@ -339,10 +394,15 @@ type DeployEnv struct { } type ArtifactArgs struct { - Name string `bson:"name" json:"name"` - ServiceName string `bson:"service_name" json:"service_name"` - Image string `bson:"image" json:"image"` - Deploy []DeployEnv `bson:"deloy" json:"deploy"` + Name string `bson:"name" json:"name"` + ImageName string `bson:"image_name,omitempty" json:"image_name,omitempty"` + ServiceName string `bson:"service_name" json:"service_name"` + Image string `bson:"image,omitempty" json:"image,omitempty"` + Deploy []DeployEnv `bson:"deploy" json:"deploy"` + WorkflowName string `bson:"workflow_name,omitempty" json:"workflow_name,omitempty"` + TaskID int64 `bson:"task_id,omitempty" json:"task_id,omitempty"` + FileName string `bson:"file_name,omitempty" json:"file_name,omitempty"` + URL string `bson:"url,omitempty" json:"url,omitempty"` } type VersionArgs struct { @@ -352,6 +412,20 @@ type VersionArgs struct { Labels []string `bson:"labels" json:"labels"` } +type BuildModuleArgs struct { + BuildName string + Target string + ServiceName string + ProductName string + Variables []*KeyVal + Env *Product + WorkflowName string + TaskID int64 + FileName string + URL string + TaskType string +} + func (Workflow) TableName() string { return "workflow" } diff --git a/pkg/microservice/aslan/core/common/repository/models/workflow_queue.go b/pkg/microservice/aslan/core/common/repository/models/workflow_queue.go new file mode 100644 index 0000000000000000000000000000000000000000..625a854d89d85b529ed496c47aed0ce069efe65c --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/models/workflow_queue.go @@ -0,0 +1,40 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" +) + +type WorkflowQueue struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + TaskID int64 `bson:"task_id" json:"task_id"` + ProjectName string `bson:"project_name" json:"project_name"` + WorkflowName string `bson:"workflow_name" json:"workflow_name"` + Status config.Status `bson:"status" json:"status,omitempty"` + Stages []*StageTask `bson:"stages" json:"stages"` + TaskCreator string `bson:"task_creator" json:"task_creator,omitempty"` + TaskRevoker string `bson:"task_revoker,omitempty" json:"task_revoker,omitempty"` + CreateTime int64 `bson:"create_time" json:"create_time,omitempty"` + MultiRun bool `bson:"multi_run" json:"multi_run"` +} + +func (WorkflowQueue) TableName() string { + return "workflow_queue" +} diff --git a/pkg/microservice/aslan/core/common/repository/models/workflow_v3.go b/pkg/microservice/aslan/core/common/repository/models/workflow_v3.go new file mode 100644 index 0000000000000000000000000000000000000000..70c6830bc167da6d0d033bbb8d00e54ebd2a6892 --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/models/workflow_v3.go @@ -0,0 +1,90 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/koderover/zadig/pkg/types" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type WorkflowV3 struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + Name string `bson:"name" json:"name"` + ProjectName string `bson:"project_name" json:"project_name"` + Description string `bson:"description" json:"description"` + Parameters []*ParameterSetting `bson:"parameters" json:"parameters"` + SubTasks []map[string]interface{} `bson:"sub_tasks" json:"sub_tasks"` + CreatedBy string `bson:"created_by" json:"created_by"` + CreateTime int64 `bson:"create_time" json:"create_time"` + UpdatedBy string `bson:"updated_by" json:"updated_by"` + UpdateTime int64 `bson:"update_time" json:"update_time"` +} + +type ParameterSettingType string + +const ( + StringType ParameterSettingType = "string" + ChoiceType ParameterSettingType = "choice" + ExternalType ParameterSettingType = "external" +) + +type ParameterSetting struct { + // External type parameter will NOT use this key. + Key string `bson:"key" json:"key"` + Type ParameterSettingType `bson:"type" json:"type"` + //DefaultValue + DefaultValue string `bson:"default_value" json:"default_value"` + // choiceOption Are all options enumerated + ChoiceOption []string `bson:"choice_option" json:"choice_option"` + // ExternalSetting It is the configuration of the external system to obtain the variable + ExternalSetting *ExternalSetting `bson:"external_setting" json:"external_setting"` +} + +type ExternalSetting struct { + SystemID string `bson:"system_id" json:"system_id"` + Endpoint string `bson:"endpoint" json:"endpoint"` + Method string `bson:"method" json:"method"` + Headers []*KV `bson:"headers" json:"headers"` + Body string `bson:"body" json:"body"` + Params []*ExternalParamMapping `bson:"params" json:"params"` +} + +type KV struct { + Key string `bson:"key" json:"key"` + Value string `bson:"value" json:"value"` +} + +type ExternalParamMapping struct { + // zadig变量名称 + ParamKey string `bson:"param_key" json:"param_key"` + // 返回中的key的位置 + ResponseKey string `bson:"response_key" json:"response_key"` + Display bool `bson:"display" json:"display"` +} + +// WorkflowV3Args 工作流v3任务参数 +type WorkflowV3Args struct { + ID string `bson:"id" json:"id"` + ProjectName string `bson:"project_name" json:"project_name"` + Name string `bson:"name" json:"name"` + Builds []*types.Repository `bson:"builds" json:"builds"` + BuildArgs []*KeyVal `bson:"build_args" json:"build_args"` +} + +func (WorkflowV3) TableName() string { + return "workflow_v3" +} diff --git a/pkg/microservice/aslan/core/common/repository/models/workflow_v4.go b/pkg/microservice/aslan/core/common/repository/models/workflow_v4.go new file mode 100644 index 0000000000000000000000000000000000000000..b93dcf896a12b3b70a94f544388735f77c5ba67e --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/models/workflow_v4.go @@ -0,0 +1,162 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "fmt" + + "go.mongodb.org/mongo-driver/bson/primitive" + "gopkg.in/yaml.v3" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/types" +) + +type WorkflowV4 struct { + ID primitive.ObjectID `bson:"_id,omitempty" yaml:"-" json:"id"` + Name string `bson:"name" yaml:"name" json:"name"` + KeyVals []*KeyVal `bson:"key_vals" yaml:"key_vals" json:"key_vals"` + Stages []*WorkflowStage `bson:"stages" yaml:"stages" json:"stages"` + Project string `bson:"project" yaml:"project" json:"project"` + Description string `bson:"description" yaml:"description" json:"description"` + CreatedBy string `bson:"created_by" yaml:"created_by" json:"created_by"` + CreateTime int64 `bson:"create_time" yaml:"create_time" json:"create_time"` + UpdatedBy string `bson:"updated_by" yaml:"updated_by" json:"updated_by"` + UpdateTime int64 `bson:"update_time" yaml:"update_time" json:"update_time"` + MultiRun bool `bson:"multi_run" yaml:"multi_run" json:"multi_run"` +} + +type WorkflowStage struct { + Name string `bson:"name" yaml:"name" json:"name"` + Parallel bool `bson:"parallel" yaml:"parallel" json:"parallel"` + Approval *Approval `bson:"approval" yaml:"approval" json:"approval"` + Jobs []*Job `bson:"jobs" yaml:"jobs" json:"jobs"` +} + +type Approval struct { + Enabled bool `bson:"enabled" yaml:"enabled" json:"enabled"` + ApproveUsers []*User `bson:"approve_users" yaml:"approve_users" json:"approve_users"` + Timeout int `bson:"timeout" yaml:"timeout" json:"timeout"` + NeededApprovers int `bson:"needed_approvers" yaml:"needed_approvers" json:"needed_approvers"` + Description string `bson:"description" yaml:"description" json:"description"` + RejectOrApprove config.ApproveOrReject `bson:"reject_or_approve" yaml:"-" json:"reject_or_approve"` +} + +type User struct { + UserID string `bson:"user_id" yaml:"user_id" json:"user_id"` + UserName string `bson:"user_name" yaml:"user_name" json:"user_name"` + RejectOrApprove config.ApproveOrReject `bson:"reject_or_approve" yaml:"-" json:"reject_or_approve"` + Comment string `bson:"comment" yaml:"-" json:"comment"` + OperationTime int64 `bson:"operation_time" yaml:"-" json:"operation_time"` +} + +type Job struct { + Name string `bson:"name" yaml:"name" json:"name"` + JobType config.JobType `bson:"type" yaml:"type" json:"type"` + Spec interface{} `bson:"spec" yaml:"spec" json:"spec"` +} + +type PluginJobSpec struct { + Properties *JobProperties `bson:"properties" yaml:"properties" json:"properties"` + Plugin *PluginTemplate `bson:"plugin" yaml:"plugin" json:"plugin"` +} + +type FreestyleJobSpec struct { + Properties *JobProperties `bson:"properties" yaml:"properties" json:"properties"` + Steps []*Step `bson:"steps" yaml:"steps" json:"steps"` + Outputs []*Output `bson:"outputs" yaml:"outputs" json:"outputs"` +} + +type ZadigBuildJobSpec struct { + DockerRegistryID string `bson:"docker_registry_id" yaml:"docker_registry_id" json:"docker_registry_id"` + ServiceAndBuilds []*ServiceAndBuild `bson:"service_and_builds" yaml:"service_and_builds" json:"service_and_builds"` +} + +type ServiceAndBuild struct { + ServiceName string `bson:"service_name" yaml:"service_name" json:"service_name"` + ServiceModule string `bson:"service_module" yaml:"service_module" json:"service_module"` + BuildName string `bson:"build_name" yaml:"build_name" json:"build_name"` + Image string `bson:"-" yaml:"-" json:"image"` + Package string `bson:"-" yaml:"-" json:"package"` + KeyVals []*KeyVal `bson:"key_vals" yaml:"key_vals" json:"key_vals"` + Repos []*types.Repository `bson:"repos" yaml:"repos" json:"repos"` +} + +type ZadigDeployJobSpec struct { + Env string `bson:"env" yaml:"env" json:"env"` + DeployType string `bson:"deploy_type" yaml:"-" json:"deploy_type"` + // fromjob/runtime, runtime 表示运行时输入，fromjob 表示从上游构建任务中获取 + Source config.DeploySourceType `bson:"source" yaml:"source" json:"source"` + // 当 source 为 fromjob 时需要，指定部署镜像来源是上游哪一个构建任务 + JobName string `bson:"job_name" yaml:"job_name" json:"job_name"` + ServiceAndImages []*ServiceAndImage `bson:"service_and_images" yaml:"-" json:"service_and_images"` +} + +type ServiceAndImage struct { + ServiceName string `bson:"service_name" yaml:"service_name" json:"service_name"` + ServiceModule string `bson:"service_module" yaml:"service_module" json:"service_module"` + Image string `bson:"image" yaml:"image" json:"image"` +} + +type JobProperties struct { + Timeout int64 `bson:"timeout" json:"timeout" yaml:"timeout"` + Retry int64 `bson:"retry" json:"retry" yaml:"retry"` + ResourceRequest setting.Request `bson:"res_req" json:"res_req" yaml:"res_req"` + ResReqSpec setting.RequestSpec `bson:"res_req_spec" json:"res_req_spec" yaml:"res_req_spec"` + ClusterID string `bson:"cluster_id" json:"cluster_id" yaml:"cluster_id"` + BuildOS string `bson:"build_os" json:"build_os" yaml:"build_os,omitempty"` + ImageFrom string `bson:"image_from" json:"image_from" yaml:"image_from,omitempty"` + ImageID string `bson:"image_id" json:"image_id" yaml:"image_id,omitempty"` + Namespace string `bson:"namespace" json:"namespace" yaml:"namespace"` + Envs []*KeyVal `bson:"envs" json:"envs" yaml:"envs,omitempty"` + Paths string `bson:"-" json:"-" yaml:"-"` + LogFileName string `bson:"log_file_name" json:"log_file_name" yaml:"log_file_name"` + DockerHost string `bson:"-" json:"docker_host,omitempty" yaml:"docker_host,omitempty"` + Cache types.Cache `bson:"cache" json:"cache" yaml:"cache"` + CacheEnable bool `bson:"cache_enable" json:"cache_enable" yaml:"cache_enable"` + CacheDirType types.CacheDirType `bson:"cache_dir_type" json:"cache_dir_type" yaml:"cache_dir_type"` + CacheUserDir string `bson:"cache_user_dir" json:"cache_user_dir" yaml:"cache_user_dir"` +} + +type Step struct { + Name string `bson:"name" json:"name" yaml:"name"` + Timeout int64 `bson:"timeout" json:"timeout" yaml:"timeout"` + StepType config.StepType `bson:"type" json:"type" yaml:"type"` + Spec interface{} `bson:"spec" json:"spec" yaml:"spec"` +} + +type Output struct { + Name string `bson:"name" json:"name" yaml:"name"` + Description string `bson:"description" json:"description" yaml:"description"` +} + +func IToiYaml(before interface{}, after interface{}) error { + b, err := yaml.Marshal(before) + if err != nil { + return fmt.Errorf("marshal task error: %v", err) + } + + if err := yaml.Unmarshal(b, after); err != nil { + return fmt.Errorf("unmarshal task error: %v", err) + } + return nil +} + +func (WorkflowV4) TableName() string { + return "workflow_v4" +} diff --git a/pkg/microservice/aslan/core/common/repository/models/yaml_template.go b/pkg/microservice/aslan/core/common/repository/models/yaml_template.go new file mode 100644 index 0000000000000000000000000000000000000000..f3347106dbee9999bc77870171c02b3602469a8d --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/models/yaml_template.go @@ -0,0 +1,35 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type YamlTemplate struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + Name string `bson:"name" json:"name"` + Content string `bson:"content" json:"content"` + Variables []*Variable `bson:"variables" json:"variables"` +} + +type Variable struct { + Key string `bson:"key" json:"key"` + Value string `bson:"value" json:"value"` +} + +func (YamlTemplate) TableName() string { + return "yaml_template" +} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/basic_images.go b/pkg/microservice/aslan/core/common/repository/mongodb/basic_images.go index 2d194a861b7041438e0c13ece018e55da70a2992..b3a9c7681cd035634b77c747ed2d135c591f91cc 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/basic_images.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/basic_images.go @@ -36,6 +36,7 @@ type BasicImageOpt struct { Value string `bson:"value"` Label string `bson:"label"` ImageFrom string `bson:"image_from"` + ImageType string `bson:"image_type"` } type BasicImageColl struct { @@ -86,6 +87,9 @@ func (c *BasicImageColl) List(opt *BasicImageOpt) ([]*models.BasicImage, error) if opt != nil && opt.Value != "" { query["value"] = opt.Value } + if opt != nil && opt.ImageType != "" { + query["image_type"] = opt.ImageType + } ctx := context.Background() resp := make([]*models.BasicImage, 0) @@ -130,6 +134,7 @@ func (c *BasicImageColl) Update(id string, args *models.BasicImage) error { "label": args.Label, "value": args.Value, "image_from": args.ImageFrom, + "image_type": args.ImageType, "update_by": args.UpdateBy, "update_time": time.Now().Unix(), }} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/build.go b/pkg/microservice/aslan/core/common/repository/mongodb/build.go index a1402eaab577b9fcda1f539c7024fb2f89670817..f19be738abc7d319fcce8811d14dc86163f11f6d 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/build.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/build.go @@ -41,6 +41,7 @@ type BuildListOption struct { BuildOS string BasicImageID string PrivateKeyID string + TemplateID string } // FindOption ... @@ -142,6 +143,9 @@ func (c *BuildColl) List(opt *BuildListOption) ([]*models.Build, error) { if len(opt.PrivateKeyID) != 0 { query["ssh.id"] = opt.PrivateKeyID } + if len(opt.TemplateID) != 0 { + query["template_id"] = opt.TemplateID + } var resp []*models.Build ctx := context.Background() @@ -266,3 +270,18 @@ func (c *BuildColl) DistinctTargets(excludeModule []string, productName string) return resp, err } + +func (c *BuildColl) GetDockerfileTemplateReference(templateID string) ([]*models.Build, error) { + ret := make([]*models.Build, 0) + query := bson.M{"post_build.docker_build.template_id": templateID} + + cursor, err := c.Collection.Find(context.TODO(), query) + if err != nil { + return nil, err + } + err = cursor.All(context.TODO(), &ret) + if err != nil { + return nil, err + } + return ret, nil +} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/build_template.go b/pkg/microservice/aslan/core/common/repository/mongodb/build_template.go new file mode 100644 index 0000000000000000000000000000000000000000..c6c888b69a62571fb1d1724f2e53b438f146459f --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/mongodb/build_template.go @@ -0,0 +1,144 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "errors" + "fmt" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type BuildTemplateQueryOption struct { + ID string + Name string +} + +type BuildTemplateColl struct { + *mongo.Collection + coll string +} + +func NewBuildTemplateColl() *BuildTemplateColl { + name := models.BuildTemplate{}.TableName() + return &BuildTemplateColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} +} + +func (c *BuildTemplateColl) GetCollectionName() string { + return c.coll +} + +func (c *BuildTemplateColl) EnsureIndex(ctx context.Context) error { + mod := mongo.IndexModel{ + Keys: bson.D{ + bson.E{Key: "name", Value: 1}, + }, + Options: options.Index().SetUnique(true), + } + _, err := c.Indexes().CreateOne(ctx, mod) + return err +} + +func (c *BuildTemplateColl) Create(obj *models.BuildTemplate) error { + if obj == nil { + return fmt.Errorf("nil object") + } + obj.ID = primitive.NilObjectID + _, err := c.InsertOne(context.TODO(), obj) + return err +} + +func (c *BuildTemplateColl) Update(idStr string, obj *models.BuildTemplate) error { + id, err := primitive.ObjectIDFromHex(idStr) + if err != nil { + return err + } + obj.ID = id + query := bson.M{"_id": id} + change := bson.M{"$set": obj} + _, err = c.UpdateOne(context.TODO(), query, change) + return err +} + +func (c *BuildTemplateColl) Find(opt *BuildTemplateQueryOption) (*models.BuildTemplate, error) { + if opt == nil { + return nil, errors.New("nil FindOption") + } + query := bson.M{} + if len(opt.ID) > 0 { + id, err := primitive.ObjectIDFromHex(opt.ID) + if err != nil { + return nil, err + } + query["_id"] = id + } + if len(opt.Name) > 0 { + query["name"] = opt.Name + } + resp := new(models.BuildTemplate) + err := c.Collection.FindOne(context.TODO(), query).Decode(resp) + if err != nil { + return nil, err + } + return resp, nil +} + +func (c *BuildTemplateColl) List(pageNum, pageSize int) ([]*models.BuildTemplate, int, error) { + resp := make([]*models.BuildTemplate, 0) + query := bson.M{} + count, err := c.CountDocuments(context.TODO(), query) + if err != nil { + return nil, 0, err + } + opt := options.Find(). + SetSkip(int64((pageNum - 1) * pageSize)). + SetLimit(int64(pageSize)) + + cursor, err := c.Collection.Find(context.TODO(), query, opt) + if err != nil { + return nil, 0, err + } + err = cursor.All(context.TODO(), &resp) + if err != nil { + return nil, 0, err + } + return resp, int(count), nil +} + +func (c *BuildTemplateColl) DeleteByName(name string) error { + query := bson.M{"name": name} + _, err := c.DeleteOne(context.TODO(), query) + return err +} + +func (c *BuildTemplateColl) DeleteByID(idStr string) error { + id, err := primitive.ObjectIDFromHex(idStr) + if err != nil { + return err + } + query := bson.M{"_id": id} + _, err = c.DeleteOne(context.TODO(), query) + return err +} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/callback_request.go b/pkg/microservice/aslan/core/common/repository/mongodb/callback_request.go new file mode 100644 index 0000000000000000000000000000000000000000..57c6a2dc77e74bd67d678ecb40a84a0e9b033700 --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/mongodb/callback_request.go @@ -0,0 +1,100 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "errors" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type CallbackRequestColl struct { + *mongo.Collection + + coll string +} + +func NewCallbackRequestColl() *CallbackRequestColl { + name := models.CallbackRequest{}.TableName() + return &CallbackRequestColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} +} + +func (c *CallbackRequestColl) GetCollectionName() string { + return c.coll +} + +func (c *CallbackRequestColl) EnsureIndex(ctx context.Context) error { + mod := mongo.IndexModel{ + Keys: bson.D{ + bson.E{Key: "task_id", Value: 1}, + bson.E{Key: "task_name", Value: 1}, + bson.E{Key: "project_name", Value: 1}, + }, + Options: options.Index().SetUnique(false), + } + + _, err := c.Indexes().CreateOne(ctx, mod) + + return err +} + +func (c *CallbackRequestColl) Create(req *models.CallbackRequest) error { + if req == nil { + return errors.New("nil callbackRequest args") + } + + _, err := c.Collection.InsertOne(context.TODO(), req) + + return err +} + +type CallbackFindOption struct { + TaskID int64 + PipelineName string + ProjectName string +} + +func (c *CallbackRequestColl) Find(req *CallbackFindOption) (*models.CallbackRequest, error) { + if req == nil { + return nil, errors.New("nil FindOption") + } + + query := bson.M{} + if req.TaskID != 0 { + query["task_id"] = req.TaskID + } + if req.PipelineName != "" { + query["task_name"] = req.PipelineName + } + if req.ProjectName != "" { + query["project_name"] = req.ProjectName + } + + resp := new(models.CallbackRequest) + err := c.Collection.FindOne(context.TODO(), query).Decode(resp) + if err != nil { + return nil, err + } + return resp, nil +} diff --git a/pkg/microservice/aslan/core/templatestore/repository/mongodb/chart.go b/pkg/microservice/aslan/core/common/repository/mongodb/chart.go similarity index 92% rename from pkg/microservice/aslan/core/templatestore/repository/mongodb/chart.go rename to pkg/microservice/aslan/core/common/repository/mongodb/chart.go index 2de408cc3fc6f176672ae4207fbfa36671d3f4bc..0fe2ed2506e41009fa0ed0f8e4401681550b06fd 100644 --- a/pkg/microservice/aslan/core/templatestore/repository/mongodb/chart.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/chart.go @@ -26,8 +26,7 @@ import ( "go.mongodb.org/mongo-driver/mongo/options" "github.com/koderover/zadig/pkg/microservice/aslan/config" - commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" - "github.com/koderover/zadig/pkg/microservice/aslan/core/templatestore/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" mongotool "github.com/koderover/zadig/pkg/tool/mongo" ) @@ -143,5 +142,5 @@ func (c *ChartColl) Delete(name string) error { func getNextRevision(name string) (int64, error) { template := fmt.Sprintf(chartTemplateCounterName, name) - return commonrepo.NewCounterColl().GetNextSeq(template) + return NewCounterColl().GetNextSeq(template) } diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/delivery_artifact.go b/pkg/microservice/aslan/core/common/repository/mongodb/delivery_artifact.go index 85cfa6f672b0e6d539c1b4b5eed15a1d6585803e..d39407a59e87c885c4a60f44d81276e56da55198 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/delivery_artifact.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/delivery_artifact.go @@ -32,19 +32,21 @@ import ( ) type DeliveryArtifactArgs struct { - ID string `json:"id"` - Image string `json:"image"` - Name string `json:"name"` - Type string `json:"type"` - RepoName string `json:"repo_name"` - Branch string `json:"branch"` - Source string `json:"source"` - ImageHash string `json:"image_hash"` - ImageTag string `json:"image_tag"` - ImageDigest string `json:"image_digest"` - PerPage int `json:"per_page"` - Page int `json:"page"` - IsFuzzyQuery bool `json:"is_fuzzy_query"` + ID string `json:"id"` + Image string `json:"image"` + Name string `json:"name"` + Type string `json:"type"` + RepoName string `json:"repo_name"` + Branch string `json:"branch"` + Source string `json:"source"` + ImageHash string `json:"image_hash"` + ImageTag string `json:"image_tag"` + ImageDigest string `json:"image_digest"` + PerPage int `json:"per_page"` + Page int `json:"page"` + IsFuzzyQuery bool `json:"is_fuzzy_query"` + OnlyCount bool `json:"only_count"` + PackageStorageURI string `json:"package_storage_uri"` } type DeliveryArtifactColl struct { @@ -119,6 +121,11 @@ func (c *DeliveryArtifactColl) List(args *DeliveryArtifactArgs) ([]*models.Deliv if err != nil { return nil, 0, fmt.Errorf("find artifact err:%v", err) } + + if args.OnlyCount { + return nil, int(count), nil + } + opt := options.Find(). SetSort(bson.D{{"created_time", -1}}). SetSkip(int64(args.PerPage * (args.Page - 1))). @@ -139,13 +146,20 @@ func (c *DeliveryArtifactColl) Get(args *DeliveryArtifactArgs) (*models.Delivery return nil, errors.New("nil delivery_artifact args") } resp := new(models.DeliveryArtifact) - id, err := primitive.ObjectIDFromHex(args.ID) - if err != nil { - return nil, err + query := bson.M{} + if args.ID != "" { + id, err := primitive.ObjectIDFromHex(args.ID) + if err != nil { + return nil, err + } + query["_id"] = id } - query := bson.M{"_id": id} - err = c.FindOne(context.TODO(), query).Decode(&resp) + if args.Image != "" { + query["image"] = args.Image + } + + err := c.FindOne(context.TODO(), query).Decode(&resp) if err != nil { return nil, err } @@ -187,3 +201,40 @@ func (c *DeliveryArtifactColl) Update(args *DeliveryArtifactArgs) error { _, err := c.UpdateOne(context.TODO(), query, change, options.Update().SetUpsert(true)) return err } + +func (c *DeliveryArtifactColl) ListTars(args *DeliveryArtifactArgs) ([]*models.DeliveryArtifact, error) { + if args == nil { + return nil, errors.New("nil delivery_artifact args") + } + + resp := make([]*models.DeliveryArtifact, 0) + query := bson.M{} + if args.Name != "" { + query["name"] = args.Name + } + + if args.Type != "" { + query["type"] = args.Type + } + + if args.Source != "" { + query["source"] = args.Source + } + + if args.PackageStorageURI != "" { + query["package_storage_uri"] = args.PackageStorageURI + } + + opt := options.Find(). + SetSort(bson.D{{"created_time", -1}}) + + cursor, err := c.Collection.Find(context.TODO(), query, opt) + if err != nil { + return nil, err + } + err = cursor.All(context.TODO(), &resp) + if err != nil { + return nil, err + } + return resp, nil +} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/delivery_distribute.go b/pkg/microservice/aslan/core/common/repository/mongodb/delivery_distribute.go index 0b3ce01a77f8ce7d851896e64ed66dc6818d7155..98a9c39ede04379452247b336046399c646ca913 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/delivery_distribute.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/delivery_distribute.go @@ -35,6 +35,7 @@ type DeliveryDistributeArgs struct { ID string `json:"id"` ReleaseID string `json:"releaseId"` ServiceName string `json:"serviceName"` + ChartName string `json:"chartName"` DistributeType config.DistributeType `json:"distributeType"` } @@ -62,6 +63,7 @@ func (c *DeliveryDistributeColl) EnsureIndex(ctx context.Context) error { Keys: bson.D{ bson.E{Key: "release_id", Value: 1}, bson.E{Key: "service_name", Value: 1}, + bson.E{Key: "chart_name", Value: 1}, bson.E{Key: "deleted_at", Value: 1}, }, Options: options.Index().SetUnique(false), @@ -115,8 +117,11 @@ func (c *DeliveryDistributeColl) Find(args *DeliveryDistributeArgs) ([]*models.D } query := bson.M{"release_id": releaseID, "deleted_at": 0} - if args.DistributeType != "" { - query["distribute_type"] = config.File + if len(args.DistributeType) > 0 { + query["distribute_type"] = string(args.DistributeType) + } + if len(args.ChartName) > 0 { + query["chart_name"] = args.ChartName } cursor, err := c.Collection.Find(context.TODO(), query) if err != nil { diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/delivery_version.go b/pkg/microservice/aslan/core/common/repository/mongodb/delivery_version.go index 69b56e10bf49d5f8fb37f109b12d69b7b975d67e..fac84960d749477243023b861228422e7a27814c 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/delivery_version.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/delivery_version.go @@ -33,8 +33,8 @@ import ( type DeliveryVersionArgs struct { ID string `json:"id"` - OrgID int `json:"orgId"` ProductName string `json:"productName"` + Version string `json:"version"` WorkflowName string `json:"workflowName"` TaskID int `json:"taskId"` PerPage int `json:"perPage"` @@ -64,6 +64,7 @@ func (c *DeliveryVersionColl) EnsureIndex(ctx context.Context) error { { Keys: bson.D{ bson.E{Key: "org_id", Value: 1}, + bson.E{Key: "product_name", Value: 1}, bson.E{Key: "workflow_name", Value: 1}, bson.E{Key: "version", Value: 1}, bson.E{Key: "deleted_at", Value: 1}, @@ -90,8 +91,7 @@ func (c *DeliveryVersionColl) Find(args *DeliveryVersionArgs) ([]*models.Deliver return nil, errors.New("nil delivery_version args") } - var resp []*models.DeliveryVersion - query := bson.M{"org_id": args.OrgID, "deleted_at": 0} + query := bson.M{"deleted_at": 0} if args.ProductName != "" { query["product_name"] = args.ProductName } @@ -114,6 +114,7 @@ func (c *DeliveryVersionColl) Find(args *DeliveryVersionArgs) ([]*models.Deliver return nil, err } + resp := make([]*models.DeliveryVersion, 0) err = cursor.All(ctx, &resp) if err != nil { return nil, err @@ -137,9 +138,9 @@ func (c *DeliveryVersionColl) Delete(id string) error { return err } -func (c *DeliveryVersionColl) ListDeliveryVersions(productName string, orgID int) ([]*models.DeliveryVersion, error) { +func (c *DeliveryVersionColl) ListDeliveryVersions(productName string) ([]*models.DeliveryVersion, error) { var resp []*models.DeliveryVersion - query := bson.M{"org_id": orgID, "deleted_at": 0} + query := bson.M{"deleted_at": 0} if productName != "" { query["product_name"] = productName } @@ -165,9 +166,15 @@ func (c *DeliveryVersionColl) Get(args *DeliveryVersionArgs) (*models.DeliveryVe resp := new(models.DeliveryVersion) var query map[string]interface{} if args.ID != "" { - query = bson.M{"_id": args.ID, "deleted_at": 0} + oid, err := primitive.ObjectIDFromHex(args.ID) + if err != nil { + return nil, err + } + query = bson.M{"_id": oid, "deleted_at": 0} + } else if len(args.Version) > 0 { + query = bson.M{"product_name": args.ProductName, "version": args.Version, "deleted_at": 0} } else { - query = bson.M{"org_id": args.OrgID, "product_name": args.ProductName, "workflow_name": args.WorkflowName, "task_id": args.TaskID, "deleted_at": 0} + query = bson.M{"product_name": args.ProductName, "workflow_name": args.WorkflowName, "task_id": args.TaskID, "deleted_at": 0} } err := c.FindOne(context.TODO(), query).Decode(&resp) @@ -191,6 +198,33 @@ func (c *DeliveryVersionColl) Insert(args *models.DeliveryVersion) error { return nil } +func (c *DeliveryVersionColl) UpdateStatusByName(versionName, projectName, status, errorStr string) error { + query := bson.M{ + "version": versionName, + "product_name": projectName, + "deleted_at": 0, + } + change := bson.M{"$set": bson.M{ + "status": status, + "error": errorStr, + }} + _, err := c.UpdateOne(context.TODO(), query, change) + return err +} + +func (c *DeliveryVersionColl) UpdateTaskID(versionName, projectName string, taskID int32) error { + query := bson.M{ + "version": versionName, + "product_name": projectName, + "deleted_at": 0, + } + change := bson.M{"$set": bson.M{ + "task_id": taskID, + }} + _, err := c.UpdateOne(context.TODO(), query, change) + return err +} + func (c *DeliveryVersionColl) Update(args *models.DeliveryVersion) error { if args == nil { return errors.New("nil delivery_version args") @@ -205,9 +239,9 @@ func (c *DeliveryVersionColl) Update(args *models.DeliveryVersion) error { return err } -func (c *DeliveryVersionColl) FindProducts(orgID int) ([]string, error) { +func (c *DeliveryVersionColl) FindProducts() ([]string, error) { resp := make([]string, 0) - query := bson.M{"org_id": orgID, "deleted_at": 0} + query := bson.M{"deleted_at": 0} ret, err := c.Distinct(context.TODO(), "product_name", query) if err != nil { return nil, err diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/dockerfile.go b/pkg/microservice/aslan/core/common/repository/mongodb/dockerfile.go new file mode 100644 index 0000000000000000000000000000000000000000..f5e36be41f1c7640b4008ed1b8bb9535e5e5d9d4 --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/mongodb/dockerfile.go @@ -0,0 +1,132 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "fmt" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type DockerfileTemplateColl struct { + *mongo.Collection + + coll string +} + +func NewDockerfileTemplateColl() *DockerfileTemplateColl { + name := models.DockerfileTemplate{}.TableName() + return &DockerfileTemplateColl{ + Collection: mongotool.Database(config.MongoDatabase()).Collection(name), + coll: name, + } +} + +func (c *DockerfileTemplateColl) GetCollectionName() string { + return c.coll +} + +func (c *DockerfileTemplateColl) EnsureIndex(ctx context.Context) error { + mod := mongo.IndexModel{ + Keys: bson.M{"name": 1}, + Options: options.Index().SetUnique(true), + } + + _, err := c.Indexes().CreateOne(ctx, mod) + + return err +} + +func (c *DockerfileTemplateColl) Create(obj *models.DockerfileTemplate) error { + if obj == nil { + return fmt.Errorf("nil object") + } + + _, err := c.InsertOne(context.TODO(), obj) + return err +} + +func (c *DockerfileTemplateColl) Update(idString string, obj *models.DockerfileTemplate) error { + if obj == nil { + return fmt.Errorf("nil object") + } + id, err := primitive.ObjectIDFromHex(idString) + if err != nil { + return fmt.Errorf("invalid id") + } + filter := bson.M{"_id": id} + update := bson.M{"$set": obj} + + _, err = c.UpdateOne(context.TODO(), filter, update) + return err +} + +func (c *DockerfileTemplateColl) List(pageNum, pageSize int) ([]*models.DockerfileTemplate, int, error) { + resp := make([]*models.DockerfileTemplate, 0) + query := bson.M{} + count, err := c.CountDocuments(context.TODO(), query) + if err != nil { + return nil, 0, err + } + opt := options.Find(). + SetSkip(int64((pageNum - 1) * pageSize)). + SetLimit(int64(pageSize)) + + cursor, err := c.Collection.Find(context.TODO(), query, opt) + if err != nil { + return nil, 0, err + } + err = cursor.All(context.TODO(), &resp) + if err != nil { + return nil, 0, err + } + return resp, int(count), nil +} + +func (c *DockerfileTemplateColl) GetById(idstring string) (*models.DockerfileTemplate, error) { + resp := new(models.DockerfileTemplate) + id, err := primitive.ObjectIDFromHex(idstring) + if err != nil { + return nil, err + } + query := bson.M{"_id": id} + + err = c.FindOne(context.TODO(), query).Decode(&resp) + if err != nil { + return nil, err + } + return resp, nil +} + +func (c *DockerfileTemplateColl) DeleteByID(idstring string) error { + id, err := primitive.ObjectIDFromHex(idstring) + if err != nil { + return err + } + query := bson.M{"_id": id} + + _, err = c.DeleteOne(context.TODO(), query) + return err +} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/env_resource.go b/pkg/microservice/aslan/core/common/repository/mongodb/env_resource.go new file mode 100644 index 0000000000000000000000000000000000000000..4e097960f2e249a0702becfc7ee7dbb2c8bd2d27 --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/mongodb/env_resource.go @@ -0,0 +1,255 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "errors" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type EnvResourceColl struct { + *mongo.Collection + + coll string +} + +func NewEnvResourceColl() *EnvResourceColl { + name := models.EnvResource{}.TableName() + return &EnvResourceColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} +} + +func (c *EnvResourceColl) GetCollectionName() string { + return c.coll +} + +func (c *EnvResourceColl) EnsureIndex(ctx context.Context) error { + mod := mongo.IndexModel{ + Keys: bson.D{ + bson.E{Key: "name", Value: 1}, + bson.E{Key: "create_time", Value: 1}, + bson.E{Key: "env_name", Value: 1}, + bson.E{Key: "type", Value: 1}, + bson.E{Key: "product_name", Value: 1}, + }, + Options: options.Index().SetUnique(true), + } + + _, err := c.Indexes().CreateOne(ctx, mod) + + return err +} + +func (c *EnvResourceColl) Create(args *models.EnvResource) error { + if args.CreateTime == 0 { + args.CreateTime = time.Now().Unix() + } + _, err := c.InsertOne(context.TODO(), args) + return err +} + +type QueryEnvResourceOption struct { + Id string + CreateTime string + IsSort bool + ProductName string + Namespace string + EnvName string + Name string + Type string + IgnoreNotFound bool + AutoSync bool + Active bool +} + +type EnvResourceBaseData struct { + ProductName string `bson:"product_name"` + EnvName string `bson:"env_name"` + Name string `bson:"name"` + Type string `bson:"type"` +} + +type EnvResourceAggregateData struct { + ID EnvResourceBaseData `bson:"_id"` + CreateTime int64 `bson:"create_time"` +} + +func (c *EnvResourceColl) List(opt *QueryEnvResourceOption) ([]*models.EnvResource, error) { + query := bson.M{} + if len(opt.ProductName) > 0 { + query["product_name"] = opt.ProductName + } + if len(opt.Namespace) > 0 { + query["namespace"] = opt.Namespace + } + if len(opt.EnvName) > 0 { + query["env_name"] = opt.EnvName + } + if len(opt.Name) > 0 { + query["name"] = opt.Name + } + if len(opt.Type) > 0 { + query["type"] = opt.Type + } + + var resp []*models.EnvResource + ctx := context.Background() + opts := options.Find() + if opt.IsSort { + opts.SetSort(bson.D{{"create_time", -1}}) + } + cursor, err := c.Collection.Find(ctx, query, opts) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &resp) + if err != nil { + return nil, err + } + + return resp, nil +} + +func (c *EnvResourceColl) Find(opt *QueryEnvResourceOption) (*models.EnvResource, error) { + if opt == nil { + return nil, errors.New("FindEnvCfgOption cannot be nil") + } + query := bson.M{} + if len(opt.ProductName) > 0 { + query["product_name"] = opt.ProductName + } + if len(opt.Namespace) > 0 { + query["namespace"] = opt.Namespace + } + if len(opt.EnvName) > 0 { + query["env_name"] = opt.EnvName + } + if len(opt.Name) > 0 { + query["name"] = opt.Name + } + if len(opt.Type) > 0 { + query["type"] = opt.Type + } + if opt.Active { + query["deleted_at"] = 0 + } + if len(opt.Id) > 0 { + oid, err := primitive.ObjectIDFromHex(opt.Id) + if err != nil { + return nil, err + } + query["_id"] = oid + } + opts := options.FindOne() + if len(opt.CreateTime) > 0 { + query["create_time"] = opt.CreateTime + } else { + opts.SetSort(bson.D{{"create_time", -1}}) + } + + rs := &models.EnvResource{} + err := c.FindOne(context.TODO(), query, opts).Decode(&rs) + if err != nil { + if err == mongo.ErrNoDocuments && opt.IgnoreNotFound { + return nil, nil + } + return nil, err + } + return rs, err +} + +func (c *EnvResourceColl) ListLatestResource(opt *QueryEnvResourceOption) ([]*EnvResourceAggregateData, error) { + query := bson.M{} + if len(opt.ProductName) > 0 { + query["product_name"] = opt.ProductName + } + if len(opt.Namespace) > 0 { + query["namespace"] = opt.Namespace + } + if len(opt.EnvName) > 0 { + query["env_name"] = opt.EnvName + } + if len(opt.Name) > 0 { + query["name"] = opt.Name + } + if len(opt.Type) > 0 { + query["type"] = opt.Type + } + + var pipeline []bson.M + if len(query) > 0 { + pipeline = append(pipeline, bson.M{"$match": query}) + } + + pipeline = append(pipeline, + bson.M{"$sort": bson.M{"create_time": -1}}, + bson.M{ + "$group": bson.M{ + "_id": bson.M{ + "product_name": "$product_name", + "env_name": "$env_name", + "name": "$name", + "type": "$type", + }, + "create_time": bson.M{"$first": "$create_time"}, + "deleted_at": bson.M{"$first": "$deleted_at"}, + "auto_sync": bson.M{"$first": "$auto_sync"}, + }, + }, + ) + + pipeline = append(pipeline, bson.M{ + "$match": bson.M{"deleted_at": 0}, + }) + + if opt.AutoSync { + pipeline = append(pipeline, bson.M{ + "$match": bson.M{"auto_sync": true}, + }) + } + + cursor, err := c.Aggregate(context.TODO(), pipeline) + if err != nil { + return nil, err + } + + result := make([]*EnvResourceAggregateData, 0) + if err := cursor.All(context.TODO(), &result); err != nil { + return nil, err + } + return result, err +} + +func (c *EnvResourceColl) Delete(oid primitive.ObjectID) error { + query := bson.M{} + query["_id"] = oid + change := bson.M{"$set": bson.M{ + "deleted_at": time.Now().Unix(), + }} + _, err := c.UpdateOne(context.TODO(), query, change) + return err +} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/env_svc_depend.go b/pkg/microservice/aslan/core/common/repository/mongodb/env_svc_depend.go new file mode 100644 index 0000000000000000000000000000000000000000..d845bbe8ecc0b5950b3705f33a47f4b2fbfcc1e5 --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/mongodb/env_svc_depend.go @@ -0,0 +1,254 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "errors" + "time" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +type EnvSvcDependColl struct { + *mongo.Collection + + coll string +} + +func NewEnvSvcDependColl() *EnvSvcDependColl { + name := models.EnvSvcDepend{}.TableName() + return &EnvSvcDependColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} +} + +func (c *EnvSvcDependColl) GetCollectionName() string { + return c.coll +} + +func (c *EnvSvcDependColl) EnsureIndex(ctx context.Context) error { + mod := mongo.IndexModel{ + Keys: bson.D{ + bson.E{Key: "name", Value: 1}, + bson.E{Key: "service_name", Value: 1}, + bson.E{Key: "env_name", Value: 1}, + bson.E{Key: "product_name", Value: 1}, + bson.E{Key: "service_module", Value: 1}, + }, + Options: options.Index().SetUnique(true), + } + + _, err := c.Indexes().CreateOne(ctx, mod) + + return err +} + +func (c *EnvSvcDependColl) Create(args *models.EnvSvcDepend) error { + + args.CreateTime = time.Now().Unix() + _, err := c.InsertOne(context.TODO(), args) + return err +} + +type ListEnvSvcDependOption struct { + IsSort bool + ProductName string + Namespace string + EnvName string + ServiceName string + ServiceModule string +} + +func (c *EnvSvcDependColl) List(opt *ListEnvSvcDependOption) ([]*models.EnvSvcDepend, error) { + query := bson.M{} + if len(opt.ProductName) > 0 { + query["product_name"] = opt.ProductName + } + if len(opt.Namespace) > 0 { + query["namespace"] = opt.Namespace + } + if len(opt.EnvName) > 0 { + query["env_name"] = opt.EnvName + } + if len(opt.ServiceName) > 0 { + query["service_name"] = opt.ServiceName + } + if len(opt.ServiceModule) > 0 { + query["service_module"] = opt.ServiceModule + } + + var resp []*models.EnvSvcDepend + ctx := context.Background() + opts := options.Find() + if opt.IsSort { + opts.SetSort(bson.D{{"create_time", -1}}) + } + cursor, err := c.Collection.Find(ctx, query, opts) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &resp) + if err != nil { + return nil, err + } + + return resp, nil +} + +func (c *EnvSvcDependColl) Update(id string, envSvcDepend *models.EnvSvcDepend) error { + oid, err := primitive.ObjectIDFromHex(id) + if err != nil { + return err + } + + query := bson.M{"_id": oid} + change := bson.M{"$set": bson.M{ + "configmaps": envSvcDepend.ConfigMaps, + "pvcs": envSvcDepend.Pvcs, + "secrets": envSvcDepend.Secrets, + }} + + _, err = c.UpdateOne(context.TODO(), query, change, options.Update().SetUpsert(true)) + return err +} + +func (c *EnvSvcDependColl) CreateOrUpdate(envSvcDepend *models.EnvSvcDepend) error { + opt := &FindEnvSvcDependOption{ + ProductName: envSvcDepend.ProductName, + EnvName: envSvcDepend.EnvName, + ServiceModule: envSvcDepend.ServiceModule, + } + envSvcDependOlder, err := NewEnvSvcDependColl().Find(opt) + if err != nil { + if IsErrNoDocuments(err) { + return NewEnvSvcDependColl().Create(envSvcDepend) + } + return err + } + + oid, err := primitive.ObjectIDFromHex(envSvcDependOlder.ID.Hex()) + if err != nil { + return err + } + + query := bson.M{"_id": oid} + change := bson.M{"$set": bson.M{ + "configmaps": envSvcDepend.ConfigMaps, + "pvcs": envSvcDepend.Pvcs, + "secrets": envSvcDepend.Secrets, + }} + + _, err = c.UpdateOne(context.TODO(), query, change, options.Update().SetUpsert(true)) + return err +} + +type FindEnvSvcDependOption struct { + Id string + CreateTime string + ProductName string + Namespace string + EnvName string + ServiceName string + ServiceModule string +} + +func (c *EnvSvcDependColl) Find(opt *FindEnvSvcDependOption) (*models.EnvSvcDepend, error) { + if opt == nil { + return nil, errors.New("FindEnvSvcDependOption cannot be nil") + } + query := bson.M{} + if len(opt.ProductName) > 0 { + query["product_name"] = opt.ProductName + } + if len(opt.Namespace) > 0 { + query["namespace"] = opt.Namespace + } + if len(opt.EnvName) > 0 { + query["env_name"] = opt.EnvName + } + if len(opt.ServiceName) > 0 { + query["service_name"] = opt.ServiceName + } + if len(opt.ServiceModule) > 0 { + query["service_module"] = opt.ServiceModule + } + if len(opt.Id) > 0 { + oid, err := primitive.ObjectIDFromHex(opt.Id) + if err != nil { + return nil, err + } + query["_id"] = oid + } + opts := options.FindOne() + if len(opt.CreateTime) > 0 { + query["create_time"] = opt.CreateTime + } else { + opts.SetSort(bson.D{{"create_time", -1}}) + } + + rs := &models.EnvSvcDepend{} + err := c.FindOne(context.TODO(), query, opts).Decode(&rs) + if err != nil { + return nil, err + } + return rs, err +} + +type DeleteEnvSvcDependOption struct { + Id string + CreateTime string + ProductName string + Namespace string + EnvName string + ServiceName string + ServiceModule string +} + +func (c *EnvSvcDependColl) Delete(opt *DeleteEnvSvcDependOption) error { + query := bson.M{} + if len(opt.ProductName) > 0 { + query["product_name"] = opt.ProductName + } + if len(opt.Namespace) > 0 { + query["namespace"] = opt.Namespace + } + if len(opt.EnvName) > 0 { + query["env_name"] = opt.EnvName + } + if len(opt.ServiceName) > 0 { + query["service_name"] = opt.ServiceName + } + if len(opt.ServiceModule) > 0 { + query["service_module"] = opt.ServiceModule + } + if len(opt.Id) > 0 { + oid, err := primitive.ObjectIDFromHex(opt.Id) + if err != nil { + return err + } + query["_id"] = oid + } + + _, err := c.DeleteOne(context.TODO(), query) + return err +} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/external_link.go b/pkg/microservice/aslan/core/common/repository/mongodb/external_link.go new file mode 100644 index 0000000000000000000000000000000000000000..414ace19dfc90c39b851f3a178e1387c0ac6f881 --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/mongodb/external_link.go @@ -0,0 +1,127 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "errors" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type ExternalLinkColl struct { + *mongo.Collection + + coll string +} + +func NewExternalLinkColl() *ExternalLinkColl { + name := models.ExternalLink{}.TableName() + return &ExternalLinkColl{ + Collection: mongotool.Database(config.MongoDatabase()).Collection(name), + coll: name, + } +} + +func (c *ExternalLinkColl) GetCollectionName() string { + return c.coll +} + +func (c *ExternalLinkColl) EnsureIndex(ctx context.Context) error { + mod := mongo.IndexModel{ + Keys: bson.D{ + bson.E{Key: "name", Value: 1}, + bson.E{Key: "url", Value: 1}, + }, + Options: options.Index().SetUnique(true), + } + _, err := c.Indexes().CreateOne(ctx, mod) + return err +} + +func (c *ExternalLinkColl) List() ([]*models.ExternalLink, error) { + query := bson.M{} + resp := make([]*models.ExternalLink, 0) + ctx := context.Background() + + cursor, err := c.Collection.Find(ctx, query) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &resp) + if err != nil { + return nil, err + } + + return resp, err +} + +func (c *ExternalLinkColl) Create(args *models.ExternalLink) error { + if args == nil { + return errors.New("nil externalLink info") + } + + args.CreateTime = time.Now().Unix() + args.UpdateTime = time.Now().Unix() + + _, err := c.InsertOne(context.TODO(), args) + + return err +} + +func (c *ExternalLinkColl) Update(id string, args *models.ExternalLink) error { + if args == nil { + return errors.New("nil externalLink info") + } + + oid, err := primitive.ObjectIDFromHex(id) + if err != nil { + return err + } + + query := bson.M{"_id": oid} + change := bson.M{"$set": bson.M{ + "name": args.Name, + "url": args.URL, + "update_by": args.UpdateBy, + "update_time": time.Now().Unix(), + }} + + _, err = c.UpdateOne(context.TODO(), query, change) + return err +} + +func (c *ExternalLinkColl) Delete(id string) error { + oid, err := primitive.ObjectIDFromHex(id) + if err != nil { + return err + } + + query := bson.M{"_id": oid} + + _, err = c.DeleteOne(context.TODO(), query) + return err +} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/external_system.go b/pkg/microservice/aslan/core/common/repository/mongodb/external_system.go new file mode 100644 index 0000000000000000000000000000000000000000..3fd2566f06d09fdb0ade105695101394c2971cc2 --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/mongodb/external_system.go @@ -0,0 +1,131 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "errors" + "fmt" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type ExternalSystemColl struct { + *mongo.Collection + + coll string +} + +func NewExternalSystemColl() *ExternalSystemColl { + name := models.ExternalSystem{}.TableName() + return &ExternalSystemColl{ + Collection: mongotool.Database(config.MongoDatabase()).Collection(name), + coll: name, + } +} + +func (c *ExternalSystemColl) GetCollectionName() string { + return c.coll +} + +func (c *ExternalSystemColl) EnsureIndex(ctx context.Context) error { + return nil +} + +func (c *ExternalSystemColl) Create(args *models.ExternalSystem) error { + if args == nil { + return errors.New("external system is nil") + } + + _, err := c.InsertOne(context.TODO(), args) + + return err +} + +func (c *ExternalSystemColl) List(pageNum, pageSize int64) ([]*models.ExternalSystem, int64, error) { + query := bson.M{} + resp := make([]*models.ExternalSystem, 0) + ctx := context.Background() + + opt := options.Find(). + SetSkip((pageNum - 1) * pageSize). + SetLimit(pageSize) + + cursor, err := c.Collection.Find(ctx, query, opt) + if err != nil { + return nil, 0, err + } + + err = cursor.All(ctx, &resp) + if err != nil { + return nil, 0, err + } + count, err := c.Collection.CountDocuments(ctx, query) + if err != nil { + return nil, 0, err + } + + return resp, count, nil +} + +func (c *ExternalSystemColl) GetByID(idstring string) (*models.ExternalSystem, error) { + resp := new(models.ExternalSystem) + id, err := primitive.ObjectIDFromHex(idstring) + if err != nil { + return nil, err + } + query := bson.M{"_id": id} + + err = c.FindOne(context.TODO(), query).Decode(&resp) + if err != nil { + return nil, err + } + return resp, nil +} + +func (c *ExternalSystemColl) Update(idString string, obj *models.ExternalSystem) error { + if obj == nil { + return fmt.Errorf("nil object") + } + id, err := primitive.ObjectIDFromHex(idString) + if err != nil { + return fmt.Errorf("invalid id") + } + filter := bson.M{"_id": id} + update := bson.M{"$set": obj} + + _, err = c.UpdateOne(context.TODO(), filter, update) + return err +} + +func (c *ExternalSystemColl) DeleteByID(idstring string) error { + id, err := primitive.ObjectIDFromHex(idstring) + if err != nil { + return err + } + query := bson.M{"_id": id} + + _, err = c.DeleteOne(context.TODO(), query) + return err +} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/favorite_pipeline.go b/pkg/microservice/aslan/core/common/repository/mongodb/favorite_pipeline.go index fb20f134748604301cbf0464e429ecf3274de9a4..54e747df64b3c26bee063077165294612f9f9b4c 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/favorite_pipeline.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/favorite_pipeline.go @@ -31,7 +31,7 @@ import ( ) type FavoriteArgs struct { - UserID int + UserID string ProductName string Name string Type string @@ -99,7 +99,7 @@ func (c *FavoriteColl) List(args *FavoriteArgs) ([]*models.Favorite, error) { return resp, err } -func (c *FavoriteColl) Find(userID int, name, Type string) (*models.Favorite, error) { +func (c *FavoriteColl) Find(userID, name, Type string) (*models.Favorite, error) { resp := new(models.Favorite) query := bson.M{"user_id": userID, "name": name} if Type != "" { diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/helm_repo.go b/pkg/microservice/aslan/core/common/repository/mongodb/helm_repo.go index 6d7731676f098c7856ba584d058b0e291bbe1745..86fff875ecd553c15ebee5683c494086ea901a18 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/helm_repo.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/helm_repo.go @@ -37,6 +37,11 @@ type HelmRepoColl struct { coll string } +type HelmRepoFindOption struct { + Id string + RepoName string +} + func NewHelmRepoColl() *HelmRepoColl { name := models.HelmRepo{}.TableName() coll := &HelmRepoColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} @@ -49,7 +54,12 @@ func (c *HelmRepoColl) GetCollectionName() string { } func (c *HelmRepoColl) EnsureIndex(ctx context.Context) error { - return nil + mod := mongo.IndexModel{ + Keys: bson.M{"repo_name": 1}, + Options: options.Index().SetUnique(false), + } + _, err := c.Indexes().CreateOne(ctx, mod) + return err } func (c *HelmRepoColl) Create(args *models.HelmRepo) error { @@ -64,6 +74,26 @@ func (c *HelmRepoColl) Create(args *models.HelmRepo) error { return err } +func (c *HelmRepoColl) Find(opt *HelmRepoFindOption) (*models.HelmRepo, error) { + query := bson.M{} + if len(opt.Id) > 0 { + oid, err := primitive.ObjectIDFromHex(opt.Id) + if err != nil { + return nil, err + } + query["_id"] = oid + } + if len(opt.RepoName) > 0 { + query["repo_name"] = opt.RepoName + } + ret := new(models.HelmRepo) + err := c.FindOne(context.TODO(), query).Decode(ret) + if err != nil { + return nil, err + } + return ret, nil +} + func (c *HelmRepoColl) Update(id string, args *models.HelmRepo) error { oid, err := primitive.ObjectIDFromHex(id) if err != nil { diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/jenkins.go b/pkg/microservice/aslan/core/common/repository/mongodb/jenkins.go index f1bbeaa948fd863decf164a7f9a63fbe18dcaacd..1cd17bcceb54b1fe0b2814e408e10019b2cdb340 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/jenkins.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/jenkins.go @@ -52,6 +52,19 @@ func (c *JenkinsIntegrationColl) EnsureIndex(ctx context.Context) error { return nil } +func (c *JenkinsIntegrationColl) Get(id string) (*models.JenkinsIntegration, error) { + oid, err := primitive.ObjectIDFromHex(id) + if err != nil { + return nil, err + } + + query := bson.M{"_id": oid} + res := &models.JenkinsIntegration{} + err = c.FindOne(context.TODO(), query).Decode(res) + + return res, err +} + func (c *JenkinsIntegrationColl) Create(args *models.JenkinsIntegration) error { if args == nil { return errors.New("nil jenkins integration args") diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/k8s_cluster.go b/pkg/microservice/aslan/core/common/repository/mongodb/k8s_cluster.go index 31f5efc8dd490178dca465d566214cd94a9daae5..b092550fe22fa1e0e5d211f4b8afe0e1f4a68b95 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/k8s_cluster.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/k8s_cluster.go @@ -30,6 +30,10 @@ import ( mongotool "github.com/koderover/zadig/pkg/tool/mongo" ) +type ClusterListOpts struct { + IDs []string +} + type K8SClusterColl struct { *mongo.Collection @@ -63,7 +67,14 @@ func (c *K8SClusterColl) Delete(id string) error { return err } -func (c *K8SClusterColl) Create(cluster *models.K8SCluster) error { +func (c *K8SClusterColl) Create(cluster *models.K8SCluster, id string) error { + if id != "" { + cluster.ID, _ = primitive.ObjectIDFromHex(id) + // If the local cluster already exists, do not insert，and return nil + if _, err := c.Get(id); err == nil { + return nil + } + } res, err := c.InsertOne(context.TODO(), cluster) if oid, ok := res.InsertedID.(primitive.ObjectID); ok { cluster.ID = oid @@ -108,6 +119,51 @@ func (c *K8SClusterColl) HasDuplicateName(id, name string) (bool, error) { return count > 0, nil } +func (c *K8SClusterColl) Count() (int64, error) { + query := bson.M{} + + ctx := context.Background() + count, err := c.Collection.CountDocuments(ctx, query) + if err != nil { + return 0, err + } + + return count, nil +} + +func (c *K8SClusterColl) List(opts *ClusterListOpts) ([]*models.K8SCluster, error) { + var clusters []*models.K8SCluster + + if opts == nil { + opts = &ClusterListOpts{} + } + + query := bson.M{} + if len(opts.IDs) > 0 { + var ids []primitive.ObjectID + for _, id := range opts.IDs { + oid, err := primitive.ObjectIDFromHex(id) + if err != nil { + return nil, err + } + ids = append(ids, oid) + } + + query["_id"] = bson.M{"$in": ids} + } + + cursor, err := c.Collection.Find(context.TODO(), query) + if err != nil { + return nil, err + } + err = cursor.All(context.TODO(), &clusters) + if err != nil { + return nil, err + } + + return clusters, err +} + func (c *K8SClusterColl) Find(clusterType string) ([]*models.K8SCluster, error) { var clusters []*models.K8SCluster @@ -140,14 +196,22 @@ func (c *K8SClusterColl) FindByName(name string) (*models.K8SCluster, error) { return res, err } -func (c *K8SClusterColl) UpdateMutableFields(cluster *models.K8SCluster) error { - _, err := c.UpdateOne(context.TODO(), +func (c *K8SClusterColl) UpdateMutableFields(cluster *models.K8SCluster, id string) error { + var err error + cluster.ID, err = primitive.ObjectIDFromHex(id) + if err != nil { + return err + } + _, err = c.UpdateOne(context.TODO(), bson.M{"_id": cluster.ID}, bson.M{"$set": bson.M{ - "name": cluster.Name, - "description": cluster.Description, - "tags": cluster.Tags, - "namespace": cluster.Namespace, - "production": cluster.Production, + "name": cluster.Name, + "description": cluster.Description, + "tags": cluster.Tags, + "namespace": cluster.Namespace, + "production": cluster.Production, + "advanced_config": cluster.AdvancedConfig, + "cache": cluster.Cache, + "dind_cfg": cluster.DindCfg, }}, ) @@ -164,6 +228,19 @@ func (c *K8SClusterColl) UpdateStatus(cluster *models.K8SCluster) error { return err } +func (c *K8SClusterColl) UpdateUpgradeAgentInfo(id, updateHubagentErrorMsg string) error { + clusterID, err := primitive.ObjectIDFromHex(id) + if err != nil { + return err + } + _, err = c.UpdateOne(context.TODO(), + bson.M{"_id": clusterID}, bson.M{"$set": bson.M{ + "update_hubagent_error_msg": updateHubagentErrorMsg, + }}, + ) + return err +} + func (c *K8SClusterColl) FindConnectedClusters() ([]*models.K8SCluster, error) { var clusters []*models.K8SCluster cursor, err := c.Collection.Find(context.TODO(), bson.M{"disconnected": false}) @@ -184,9 +261,9 @@ func (c *K8SClusterColl) UpdateConnectState(id string, disconnected bool) error newState := bson.M{"disconnected": disconnected} if disconnected { - newState["status"] = config.Disconnected + newState["status"] = string(setting.Disconnected) } else { - newState["status"] = config.Pending + newState["status"] = string(setting.Pending) } _, err = c.UpdateMany(context.TODO(), bson.M{"_id": oid}, bson.M{"$set": newState}) @@ -201,3 +278,21 @@ func (c *K8SClusterColl) GetByToken(token string) (*models.K8SCluster, error) { return c.Get(id) } + +func (c *K8SClusterColl) FindActiveClusters() ([]*models.K8SCluster, error) { + connectedClusters, err := c.FindConnectedClusters() + if err != nil { + return nil, err + } + + activeClusters := []*models.K8SCluster{} + for _, cluster := range connectedClusters { + if cluster.Status != setting.Normal { + continue + } + + activeClusters = append(activeClusters, cluster) + } + + return activeClusters, nil +} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/plugin_repo.go b/pkg/microservice/aslan/core/common/repository/mongodb/plugin_repo.go new file mode 100644 index 0000000000000000000000000000000000000000..064e090d5703fa94847b5114e4ab0462be42075b --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/mongodb/plugin_repo.go @@ -0,0 +1,85 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type PluginRepoColl struct { + *mongo.Collection + + coll string +} + +func NewPluginRepoColl() *PluginRepoColl { + name := models.PluginRepo{}.TableName() + return &PluginRepoColl{ + Collection: mongotool.Database(config.MongoDatabase()).Collection(name), + coll: name, + } +} + +func (c *PluginRepoColl) EnsureIndex(ctx context.Context) error { + return nil +} + +func (c *PluginRepoColl) GetCollectionName() string { + return c.coll +} + +func (c *PluginRepoColl) List() ([]*models.PluginRepo, error) { + var res []*models.PluginRepo + query := bson.M{} + cursor, err := c.Collection.Find(context.TODO(), query) + if err != nil { + return nil, err + } + err = cursor.All(context.TODO(), &res) + + return res, err +} + +func (c *PluginRepoColl) Delete(id string) error { + oid, err := primitive.ObjectIDFromHex(id) + if err != nil { + return err + } + query := bson.M{"_id": oid} + + _, err = c.DeleteOne(context.TODO(), query) + return err +} + +func (c *PluginRepoColl) Upsert(pluginRepo *models.PluginRepo) error { + query := bson.M{"repo_owner": pluginRepo.RepoOwner, "repo_name": pluginRepo.RepoName, "branch": pluginRepo.Branch} + change := bson.M{"$set": pluginRepo} + pluginRepo.UpdateTime = time.Now().Unix() + + _, err := c.UpdateOne(context.TODO(), query, change, options.Update().SetUpsert(true)) + return err +} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/private_key.go b/pkg/microservice/aslan/core/common/repository/mongodb/private_key.go index 639b7b7300dcc60f419aba38defdf72920790047..cf87ff60f2038e087624b0d8b620d5d986207638 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/private_key.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/private_key.go @@ -32,7 +32,9 @@ import ( ) type PrivateKeyArgs struct { - Name string + Name string + ProjectName string + SystemOnly bool } type PrivateKeyColl struct { @@ -54,12 +56,18 @@ func (c *PrivateKeyColl) GetCollectionName() string { } func (c *PrivateKeyColl) EnsureIndex(ctx context.Context) error { - mod := mongo.IndexModel{ - Keys: bson.M{"label": 1}, - Options: options.Index().SetUnique(false), + mods := []mongo.IndexModel{ + { + Keys: bson.M{"label": 1}, + Options: options.Index().SetUnique(false), + }, + { + Keys: bson.M{"name": 1}, + Options: options.Index().SetUnique(true), + }, } - _, err := c.Indexes().CreateOne(ctx, mod) + _, err := c.Indexes().CreateMany(ctx, mods) return err } @@ -91,6 +99,14 @@ func (c *PrivateKeyColl) List(args *PrivateKeyArgs) ([]*models.PrivateKey, error if args.Name != "" { query["name"] = args.Name } + + if args.ProjectName != "" { + query["project_name"] = args.ProjectName + } + if args.SystemOnly { + query["project_name"] = bson.M{"$exists": false} + } + resp := make([]*models.PrivateKey, 0) ctx := context.Background() @@ -131,16 +147,26 @@ func (c *PrivateKeyColl) Update(id string, args *models.PrivateKey) error { } query := bson.M{"_id": oid} - change := bson.M{"$set": bson.M{ - "name": args.Name, - "user_name": args.UserName, - "ip": args.IP, - "label": args.Label, - "is_prod": args.IsProd, - "private_key": args.PrivateKey, - "update_by": args.UpdateBy, - "update_time": time.Now().Unix(), - }} + change := bson.M{} + if args.UpdateStatus { + change = bson.M{"$set": bson.M{ + "status": args.Status, + }} + } else { + change = bson.M{"$set": bson.M{ + "name": args.Name, + "user_name": args.UserName, + "ip": args.IP, + "port": args.Port, + "label": args.Label, + "is_prod": args.IsProd, + "private_key": args.PrivateKey, + "provider": args.Provider, + "probe": args.Probe, + "update_by": args.UpdateBy, + "update_time": time.Now().Unix(), + }} + } _, err = c.UpdateOne(context.TODO(), query, change) return err @@ -158,23 +184,55 @@ func (c *PrivateKeyColl) Delete(id string) error { return err } -func (c *PrivateKeyColl) ListHostIPByIDs(ids []string) ([]*models.PrivateKey, error) { +func (c *PrivateKeyColl) BulkDelete(projectName string) error { + query := bson.M{} + query["project_name"] = projectName + + _, err := c.DeleteMany(context.TODO(), query) + return err +} + +func (c *PrivateKeyColl) DeleteAll() error { + _, err := c.DeleteMany(context.TODO(), bson.M{}) + return err +} + +type ListHostIPArgs struct { + IDs []string `json:"ids"` + Labels []string `json:"labels"` +} + +func (c *PrivateKeyColl) ListHostIPByArgs(args *ListHostIPArgs) ([]*models.PrivateKey, error) { query := bson.M{} resp := make([]*models.PrivateKey, 0) ctx := context.Background() - var oids []primitive.ObjectID - for _, id := range ids { - oid, err := primitive.ObjectIDFromHex(id) - if err != nil { - return nil, err + if len(args.IDs) == 0 && len(args.Labels) == 0 { + return resp, nil + } + + if len(args.IDs) > 0 { + var oids []primitive.ObjectID + for _, id := range args.IDs { + oid, err := primitive.ObjectIDFromHex(id) + if err != nil { + return nil, err + } + oids = append(oids, oid) } - oids = append(oids, oid) + query["_id"] = bson.M{"$in": oids} + } else if len(args.Labels) > 0 { + query["label"] = bson.M{"$in": args.Labels} } - query["_id"] = bson.M{"$in": oids} + opt := options.Find() selector := bson.D{ + {"_id", 1}, {"ip", 1}, + {"port", 1}, + {"name", 1}, + {"user_name", 1}, + {"private_key", 1}, } opt.SetProjection(selector) cursor, err := c.Collection.Find(ctx, query, opt) @@ -189,3 +247,22 @@ func (c *PrivateKeyColl) ListHostIPByIDs(ids []string) ([]*models.PrivateKey, er return resp, err } + +// DistinctLabels returns distinct label +func (c *PrivateKeyColl) DistinctLabels() ([]string, error) { + var resp []string + query := bson.M{} + ctx := context.Background() + labels, err := c.Collection.Distinct(ctx, "label", query) + + for _, labelInter := range labels { + if label, ok := labelInter.(string); ok { + if label == "" { + continue + } + resp = append(resp, label) + } + } + + return resp, err +} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/product.go b/pkg/microservice/aslan/core/common/repository/mongodb/product.go index 564fdc796af64d4ddead57520574507b08697c00..9fc86e1d889fddfebd91be54d65e53ea335f7fb0 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/product.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/product.go @@ -23,8 +23,10 @@ import ( "time" "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" + "k8s.io/apimachinery/pkg/util/sets" "github.com/koderover/zadig/pkg/microservice/aslan/config" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" @@ -33,21 +35,31 @@ import ( ) type ProductFindOptions struct { - Name string - EnvName string + Name string + EnvName string + Namespace string } // ClusterId is a primitive.ObjectID{}.Hex() type ProductListOptions struct { - EnvName string - Name string - IsPublic bool - ClusterID string - IsSort bool - ExcludeStatus string - ExcludeSource string - Source string - InProjects []string + EnvName string + Name string + Namespace string + IsPublic bool + ClusterID string + IsSortByUpdateTime bool + IsSortByProductName bool + ExcludeStatus []string + ExcludeSource string + Source string + InProjects []string + InEnvs []string + InIDs []string + + // New Since v1.11.0 + ShareEnvEnable *bool + ShareEnvIsBase *bool + ShareEnvBaseEnv *string } type projectEnvs struct { @@ -128,21 +140,80 @@ func (c *ProductColl) Find(opt *ProductFindOptions) (*models.Product, error) { if opt.EnvName != "" { query["env_name"] = opt.EnvName } + if opt.Namespace != "" { + query["namespace"] = opt.Namespace + } err := c.FindOne(context.TODO(), query).Decode(res) return res, err } +func (c *ProductColl) EnvCount() (int64, error) { + query := bson.M{"status": bson.M{"$ne": setting.ProductStatusDeleting}} + + ctx := context.Background() + count, err := c.Collection.CountDocuments(ctx, query) + if err != nil { + return 0, err + } + + return count, nil +} + +type Product struct { + Name string `json:"name"` + ProjectName string `json:"projectName"` +} + +type ListProductOpt struct { + Products []Product +} + +func (c *ProductColl) ListByProducts(opt ListProductOpt) ([]*models.Product, error) { + var res []*models.Product + + if len(opt.Products) == 0 { + return nil, nil + } + condition := bson.A{} + for _, pro := range opt.Products { + condition = append(condition, bson.M{ + "env_name": pro.Name, + "product_name": pro.ProjectName, + }) + } + filter := bson.D{{"$or", condition}} + cursor, err := c.Collection.Find(context.TODO(), filter) + if err == mongo.ErrNoDocuments { + return nil, nil + } + if err != nil { + return nil, err + } + if err := cursor.All(context.TODO(), &res); err != nil { + return nil, err + } + return res, nil +} + func (c *ProductColl) List(opt *ProductListOptions) ([]*models.Product, error) { var ret []*models.Product query := bson.M{} + if opt == nil { + opt = &ProductListOptions{} + } if opt.EnvName != "" { query["env_name"] = opt.EnvName + } else if len(opt.InEnvs) > 0 { + query["env_name"] = bson.M{"$in": opt.InEnvs} } if opt.Name != "" { query["product_name"] = opt.Name } + if opt.Namespace != "" { + query["namespace"] = opt.Namespace + } if opt.IsPublic { query["is_public"] = opt.IsPublic } @@ -155,18 +226,41 @@ func (c *ProductColl) List(opt *ProductListOptions) ([]*models.Product, error) { if opt.ExcludeSource != "" { query["source"] = bson.M{"$ne": opt.ExcludeSource} } - if opt.ExcludeStatus != "" { - query["status"] = bson.M{"$ne": opt.ExcludeStatus} + if len(opt.ExcludeStatus) > 0 { + query["status"] = bson.M{"$nin": opt.ExcludeStatus} } if len(opt.InProjects) > 0 { query["product_name"] = bson.M{"$in": opt.InProjects} } + if len(opt.InIDs) > 0 { + var oids []primitive.ObjectID + for _, id := range opt.InIDs { + oid, err := primitive.ObjectIDFromHex(id) + if err != nil { + return nil, err + } + oids = append(oids, oid) + } + query["_id"] = bson.M{"$in": oids} + } + if opt.ShareEnvEnable != nil { + query["share_env.enable"] = *opt.ShareEnvEnable + } + if opt.ShareEnvIsBase != nil { + query["share_env.is_base"] = *opt.ShareEnvIsBase + } + if opt.ShareEnvBaseEnv != nil { + query["share_env.base_env"] = *opt.ShareEnvBaseEnv + } ctx := context.Background() opts := options.Find() - if opt.IsSort { + if opt.IsSortByUpdateTime { opts.SetSort(bson.D{{"update_time", -1}}) } + if opt.IsSortByProductName { + opts.SetSort(bson.D{{"product_name", 1}}) + } cursor, err := c.Collection.Find(ctx, query, opts) if err != nil { return nil, err @@ -180,10 +274,15 @@ func (c *ProductColl) List(opt *ProductListOptions) ([]*models.Product, error) { return ret, nil } -func (c *ProductColl) ListProjects() ([]*projectEnvs, error) { +func (c *ProductColl) ListProjectsInNames(names []string) ([]*projectEnvs, error) { var res []*projectEnvs - pipeline := []bson.M{ - { + var pipeline []bson.M + if len(names) > 0 { + pipeline = append(pipeline, bson.M{"$match": bson.M{"product_name": bson.M{"$in": names}}}) + } + + pipeline = append(pipeline, + bson.M{ "$group": bson.M{ "_id": bson.M{ "product_name": "$product_name", @@ -192,7 +291,7 @@ func (c *ProductColl) ListProjects() ([]*projectEnvs, error) { "envs": bson.M{"$push": "$env_name"}, }, }, - } + ) cursor, err := c.Aggregate(context.TODO(), pipeline) if err != nil { @@ -206,6 +305,16 @@ func (c *ProductColl) ListProjects() ([]*projectEnvs, error) { return res, nil } +func (c *ProductColl) UpdateStatusAndError(envName, projectName, status, errorMsg string) error { + query := bson.M{"env_name": envName, "product_name": projectName} + change := bson.M{"$set": bson.M{ + "status": status, + "error": errorMsg, + }} + _, err := c.UpdateOne(context.TODO(), query, change) + return err +} + func (c *ProductColl) UpdateStatus(owner, productName, status string) error { query := bson.M{"env_name": owner, "product_name": productName} change := bson.M{"$set": bson.M{ @@ -226,6 +335,16 @@ func (c *ProductColl) UpdateErrors(owner, productName, errorMsg string) error { return err } +func (c *ProductColl) UpdateRegistry(envName, productName, registryId string) error { + query := bson.M{"env_name": envName, "product_name": productName} + change := bson.M{"$set": bson.M{ + "registry_id": registryId, + }} + _, err := c.UpdateOne(context.TODO(), query, change) + + return err +} + func (c *ProductColl) UpdateRender(envName, productName string, render *models.RenderInfo) error { query := bson.M{"env_name": envName, "product_name": productName} change := bson.M{"$set": bson.M{ @@ -253,6 +372,7 @@ func (c *ProductColl) Update(args *models.Product) error { "revision": args.Revision, "render": args.Render, "error": args.Error, + "share_env": args.ShareEnv, }} _, err := c.UpdateOne(context.TODO(), query, change) @@ -339,3 +459,31 @@ func (c *ProductColl) UpdateAll(envs []*models.Product) error { return err } + +type nsObject struct { + ID primitive.ObjectID `bson:"_id"` + Namespace string `bson:"namespace"` +} + +func (c *ProductColl) ListExistedNamespace() ([]string, error) { + nsList := make([]*nsObject, 0) + resp := sets.NewString() + selector := bson.D{ + {"namespace", 1}, + } + query := bson.M{"is_existed": true} + opt := options.Find() + opt.SetProjection(selector) + cursor, err := c.Collection.Find(context.TODO(), query, opt) + if err != nil { + return nil, err + } + err = cursor.All(context.TODO(), &nsList) + if err != nil { + return nil, err + } + for _, obj := range nsList { + resp.Insert(obj.Namespace) + } + return resp.List(), nil +} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/project_cluster_relation.go b/pkg/microservice/aslan/core/common/repository/mongodb/project_cluster_relation.go new file mode 100644 index 0000000000000000000000000000000000000000..00f9016ad79b850a0ad3a49a1b6e46acd04d5ecf --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/mongodb/project_cluster_relation.go @@ -0,0 +1,117 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "errors" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type ProjectClusterRelationColl struct { + *mongo.Collection + + coll string +} + +func NewProjectClusterRelationColl() *ProjectClusterRelationColl { + name := models.ProjectClusterRelation{}.TableName() + return &ProjectClusterRelationColl{ + Collection: mongotool.Database(config.MongoDatabase()).Collection(name), + coll: name, + } +} + +func (c *ProjectClusterRelationColl) GetCollectionName() string { + return c.coll +} + +func (c *ProjectClusterRelationColl) EnsureIndex(ctx context.Context) error { + mod := mongo.IndexModel{ + Keys: bson.D{ + bson.E{Key: "project_name", Value: 1}, + bson.E{Key: "cluster_id", Value: 1}, + }, + Options: options.Index().SetUnique(true), + } + + _, err := c.Indexes().CreateOne(ctx, mod) + return err +} + +func (c *ProjectClusterRelationColl) Create(args *models.ProjectClusterRelation) error { + if args == nil { + return errors.New("nil projectClusterRelation info") + } + + args.CreatedAt = time.Now().Unix() + _, err := c.InsertOne(context.TODO(), args) + + return err +} + +type ProjectClusterRelationOption struct { + ProjectName string + ClusterID string +} + +func (c *ProjectClusterRelationColl) List(opt *ProjectClusterRelationOption) ([]*models.ProjectClusterRelation, error) { + query := bson.M{} + if opt.ProjectName != "" { + query["project_name"] = opt.ProjectName + } + + if opt.ClusterID != "" { + query["cluster_id"] = opt.ClusterID + } + + var resp []*models.ProjectClusterRelation + ctx := context.Background() + cursor, err := c.Collection.Find(ctx, query) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &resp) + if err != nil { + return nil, err + } + + return resp, nil +} + +func (c *ProjectClusterRelationColl) Delete(opt *ProjectClusterRelationOption) error { + query := bson.M{} + if opt.ProjectName != "" { + query["project_name"] = opt.ProjectName + } + + if opt.ClusterID != "" { + query["cluster_id"] = opt.ClusterID + } + + _, err := c.DeleteMany(context.TODO(), query) + return err +} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/registry_namespace.go b/pkg/microservice/aslan/core/common/repository/mongodb/registry_namespace.go index 27c67248e5eae6305913f1062783056cdcb24997..d3df395c3493207a9f8ab626e3ea2b07afe7aeaa 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/registry_namespace.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/registry_namespace.go @@ -33,7 +33,6 @@ import ( type FindRegOps struct { ID string `json:"id"` - OrgID int `json:"org_id"` RegAddr string `json:"reg_addr"` RegType string `json:"reg_type"` RegProvider string `json:"reg_provider"` @@ -97,10 +96,6 @@ func (opt FindRegOps) getQuery() bson.M { query["is_default"] = true } - if opt.OrgID != 0 { - query["org_id"] = opt.OrgID - } - if opt.RegType != "" { query["reg_type"] = opt.RegType } @@ -148,28 +143,6 @@ func (r *RegistryNamespaceColl) FindAll(opt *FindRegOps) ([]*models.RegistryName return resp, err } -func (r *RegistryNamespaceColl) List(orgID int, regType string) ([]*models.RegistryNamespace, error) { - query := bson.M{"org_id": orgID} - if regType != "" { - query["reg_type"] = regType - } - - ctx := context.Background() - opts := options.Find() - resp := make([]*models.RegistryNamespace, 0) - cursor, err := r.Collection.Find(ctx, query, opts) - if err != nil { - return nil, err - } - - err = cursor.All(ctx, &resp) - if err != nil { - return nil, err - } - - return resp, err -} - func (r *RegistryNamespaceColl) Update(id string, args *models.RegistryNamespace) error { if args == nil { return errors.New("nil Install") diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/render_set.go b/pkg/microservice/aslan/core/common/repository/mongodb/render_set.go index 8778156e1bb50c49e0e402af3d25865a8942827e..d000c703385a380a2b6ec03092c3a17a307edd97 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/render_set.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/render_set.go @@ -19,6 +19,7 @@ package mongodb import ( "context" "errors" + "fmt" "time" "go.mongodb.org/mongo-driver/bson" @@ -32,15 +33,20 @@ import ( type RenderSetListOption struct { // if Revision == 0 then search max revision of RenderSet - Revision int64 - ProductTmpl string + ProductTmpl string + Revisions []int64 + RendersetName string + FindOpts []RenderSetFindOption } // RenderSetFindOption ... type RenderSetFindOption struct { // if Revision == 0 then search max revision of RenderSet - Revision int64 - Name string + ProductTmpl string + EnvName string + IsDefault bool + Revision int64 + Name string } type RenderSetPipeResp struct { @@ -80,6 +86,64 @@ func (c *RenderSetColl) EnsureIndex(ctx context.Context) error { return err } +func (c *RenderSetColl) ListRendersets(opt *RenderSetListOption) ([]*models.RenderSet, error) { + query := bson.M{} + if len(opt.ProductTmpl) > 0 { + query["product_tmpl"] = opt.ProductTmpl + } + if len(opt.RendersetName) > 0 { + query["name"] = opt.RendersetName + } + if len(opt.Revisions) > 0 { + query["revision"] = bson.M{"$in": opt.Revisions} + } + + if len(query) == 0 { + return nil, fmt.Errorf("query with no filter is not allowed") + } + + var rendersets []*models.RenderSet + cursor, err := c.Collection.Find(context.TODO(), query) + if err != nil { + return nil, err + } + err = cursor.All(context.TODO(), &rendersets) + if err != nil { + return nil, err + } + + return rendersets, err + +} + +func (c *RenderSetColl) ListByFindOpts(opt *RenderSetListOption) ([]models.RenderSet, error) { + var resp []models.RenderSet + condition := bson.A{} + if len(opt.FindOpts) == 0 { + return nil, nil + } + for _, findOpt := range opt.FindOpts { + condition = append(condition, bson.M{ + "name": findOpt.Name, + "revision": findOpt.Revision, + }) + } + projectCon := bson.A{} + projectCon = append(projectCon, bson.M{"product_tmpl": opt.ProductTmpl}) + filter := bson.D{{"$or", condition}, {"$and", projectCon}} + cursor, err := c.Collection.Find(context.TODO(), filter) + if err == mongo.ErrNoDocuments { + return nil, nil + } + if err != nil { + return nil, err + } + if err := cursor.All(context.TODO(), &resp); err != nil { + return nil, err + } + return resp, nil +} + func (c *RenderSetColl) List(opt *RenderSetListOption) ([]*models.RenderSet, error) { var pipeResp []*RenderSetPipeResp var pipeline []bson.M @@ -107,8 +171,9 @@ func (c *RenderSetColl) List(opt *RenderSetListOption) ([]*models.RenderSet, err var resp []*models.RenderSet for _, pipe := range pipeResp { optRender := &RenderSetFindOption{ - Revision: pipe.Revision, - Name: pipe.RenderSet.Name, + Revision: pipe.Revision, + Name: pipe.RenderSet.Name, + ProductTmpl: pipe.RenderSet.ProductTmpl, } if pipe.Revision == 0 && pipe.RenderSet.Name == "" { continue @@ -142,9 +207,24 @@ func (c *RenderSetColl) Find(opt *RenderSetFindOption) (*models.RenderSet, error query := bson.M{"name": opt.Name} opts := options.FindOne() if opt.Revision > 0 { + // revisionName + revision are enough to locate the target record + // there is no need to set other query condition + // Note. the query logic has been rolled back to 1.12.0 query["revision"] = opt.Revision } else { opts.SetSort(bson.D{{"revision", -1}}) + + if len(opt.EnvName) > 0 { + query["env_name"] = opt.EnvName + } + + if len(opt.ProductTmpl) > 0 { + query["product_tmpl"] = opt.ProductTmpl + } + + if opt.IsDefault { + query["is_default"] = opt.IsDefault + } } rs := &models.RenderSet{} @@ -172,6 +252,7 @@ func (c *RenderSetColl) Update(args *models.RenderSet) error { "chart_infos": args.ChartInfos, "update_time": time.Now().Unix(), "update_by": args.UpdateBy, + "kvs": args.KVs, }} _, err := c.UpdateOne(context.TODO(), query, change) diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/s3.go b/pkg/microservice/aslan/core/common/repository/mongodb/s3.go index a79e666cbcd07ca8f512f654c0b0a6c722ff0829..7b5f448abc7e91e75304d245d168adbfd87e6b93 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/s3.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/s3.go @@ -18,6 +18,8 @@ package mongodb import ( "context" + "fmt" + "strings" "time" "go.mongodb.org/mongo-driver/bson" @@ -27,6 +29,7 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/tool/crypto" + "github.com/koderover/zadig/pkg/tool/log" mongotool "github.com/koderover/zadig/pkg/tool/mongo" ) @@ -201,3 +204,46 @@ func (c *S3StorageColl) FindAll() ([]*models.S3Storage, error) { return storages, nil } + +func (c *S3StorageColl) InitData() error { + minioEndpoint := config.S3StorageEndpoint() + endpointInfo := strings.Split(minioEndpoint, ":") + if len(endpointInfo) != 2 { + return fmt.Errorf("invalid endpoint of minio: %s", minioEndpoint) + } + minioEndpoint = fmt.Sprintf("%s.%s.svc.cluster.local:%s", endpointInfo[0], config.Namespace(), endpointInfo[1]) + + // Check whether minio has been integrated. + err := c.FindOne(context.TODO(), bson.M{"endpoint": minioEndpoint}).Err() + if err == nil { + log.Infof("Has found %s.", minioEndpoint) + return nil + } + if err != mongo.ErrNoDocuments { + return fmt.Errorf("failed to operate on mongodb: %s", err) + } + + // Check whether there's default S3 system. + var setDefault bool + err = c.FindOne(context.TODO(), bson.M{"is_default": true}).Err() + if err != nil { + if err != mongo.ErrNoDocuments { + return fmt.Errorf("failed to operate on mongodb: %s", err) + } + + setDefault = true + } + + minioStorage := models.S3Storage{ + Ak: config.S3StorageAK(), + Sk: config.S3StorageSK(), + Endpoint: minioEndpoint, + Bucket: config.S3StorageBucket(), + IsDefault: setDefault, + Insecure: true, + Provider: 0, + } + log.Infof("Begin to integrate minio storage: %s", minioEndpoint) + + return c.Create(&minioStorage) +} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/scanning.go b/pkg/microservice/aslan/core/common/repository/mongodb/scanning.go new file mode 100644 index 0000000000000000000000000000000000000000..8f11cf094b80b52a7cc2819d6c2d9eb96a9ad878 --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/mongodb/scanning.go @@ -0,0 +1,160 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "errors" + "fmt" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type ScanningListOption struct { + ProjectName string +} + +type ScanningColl struct { + *mongo.Collection + + coll string +} + +func NewScanningColl() *ScanningColl { + name := models.Scanning{}.TableName() + return &ScanningColl{ + Collection: mongotool.Database(config.MongoDatabase()).Collection(name), + coll: name, + } +} + +func (c *ScanningColl) GetCollectionName() string { + return c.coll +} + +func (c *ScanningColl) EnsureIndex(ctx context.Context) error { + mod := []mongo.IndexModel{ + { + Keys: bson.D{ + bson.E{Key: "project_name", Value: 1}, + bson.E{Key: "name", Value: 1}, + }, + Options: options.Index().SetUnique(true), + }, + } + + _, err := c.Indexes().CreateMany(ctx, mod) + + return err +} + +func (c *ScanningColl) Create(scanning *models.Scanning) error { + if scanning == nil { + return errors.New("nil scanning args") + } + + scanning.CreatedAt = time.Now().Unix() + + _, err := c.InsertOne(context.TODO(), scanning) + return err +} + +func (c *ScanningColl) Update(idString string, scanning *models.Scanning) error { + if scanning == nil { + return fmt.Errorf("nil object") + } + id, err := primitive.ObjectIDFromHex(idString) + if err != nil { + return fmt.Errorf("invalid id") + } + + scanning.UpdatedAt = time.Now().Unix() + + filter := bson.M{"_id": id} + update := bson.M{"$set": scanning} + + _, err = c.UpdateOne(context.TODO(), filter, update) + return err +} + +func (c *ScanningColl) List(listOption *ScanningListOption, pageNum, pageSize int64) ([]*models.Scanning, int64, error) { + query := bson.M{} + resp := make([]*models.Scanning, 0) + ctx := context.Background() + + opt := options.Find() + + if pageNum != 0 && pageSize != 0 { + opt. + SetSkip((pageNum - 1) * pageSize). + SetLimit(pageSize) + } + + if listOption != nil { + query["project_name"] = listOption.ProjectName + } + + cursor, err := c.Collection.Find(ctx, query, opt) + if err != nil { + return nil, 0, err + } + + err = cursor.All(ctx, &resp) + if err != nil { + return nil, 0, err + } + count, err := c.Collection.CountDocuments(ctx, query) + if err != nil { + return nil, 0, err + } + + return resp, count, nil +} + +func (c *ScanningColl) GetByID(idstring string) (*models.Scanning, error) { + resp := new(models.Scanning) + id, err := primitive.ObjectIDFromHex(idstring) + if err != nil { + return nil, err + } + query := bson.M{"_id": id} + + err = c.FindOne(context.TODO(), query).Decode(&resp) + if err != nil { + return nil, err + } + return resp, nil +} + +func (c *ScanningColl) DeleteByID(idstring string) error { + id, err := primitive.ObjectIDFromHex(idstring) + if err != nil { + return err + } + query := bson.M{"_id": id} + + _, err = c.DeleteOne(context.TODO(), query) + return err +} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/service.go b/pkg/microservice/aslan/core/common/repository/mongodb/service.go index 93dcba9017a555fd8a646a237e9aca523d319713..6c48cd88aa4a8d23896485885cf287feb4447f81 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/service.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/service.go @@ -37,15 +37,16 @@ import ( ) type ServiceFindOption struct { - ServiceName string - Revision int64 - Type string - Source string - ProductName string - ExcludeStatus string - CodehostID int - RepoName string - BranchName string + ServiceName string + Revision int64 + Type string + Source string + ProductName string + ExcludeStatus string + CodehostID int + RepoName string + BranchName string + IgnoreNoDocumentErr bool } type ServiceListOption struct { @@ -60,6 +61,20 @@ type ServiceListOption struct { NotInServices []*templatemodels.ServiceInfo } +type ServiceRevision struct { + ServiceName string + Revision int64 +} + +type SvcRevisionListOption struct { + ProductName string + ServiceRevisions []*ServiceRevision +} + +type ServiceAggregateResult struct { + ServiceID primitive.ObjectID `bson:"service_id"` +} + type ServiceColl struct { *mongo.Collection @@ -160,38 +175,37 @@ func (c *ServiceColl) ListMaxRevisionsForServices(services []*templatemodels.Ser return c.listMaxRevisions(pre, post) } -// TODO refactor mouuii -// ListExternalServicesBy list service only for external services ,other service type not use before refactor -func (c *ServiceColl) ListExternalWorkloadsBy(productName, envName string) ([]*models.Service, error) { - services := make([]*models.Service, 0) - query := bson.M{ - "status": bson.M{"$ne": setting.ProductStatusDeleting}, - } - if productName != "" { - query["product_name"] = productName - } - if envName != "" { - query["env_name"] = envName +func (c *ServiceColl) ListMaxRevisionsByProduct(productName string) ([]*models.Service, error) { + m := bson.M{ + "product_name": productName, + "status": bson.M{"$ne": setting.ProductStatusDeleting}, } - ctx := context.Background() - cursor, err := c.Collection.Find(ctx, query) - if err != nil { - return nil, err + + return c.listMaxRevisions(m, nil) +} + +func (c *ServiceColl) ListMaxRevisionServicesByYamlTemplate(templateId string) ([]*models.Service, error) { + m := bson.M{ + "template_id": templateId, + "status": bson.M{"$ne": setting.ProductStatusDeleting}, + "source": setting.ServiceSourceTemplate, } + return c.listMaxRevisions(m, nil) +} - err = cursor.All(ctx, &services) - if err != nil { - return nil, err +func (c *ServiceColl) ListMaxRevisionServicesByChartTemplate(templateName string) ([]*models.Service, error) { + m := bson.M{ + "create_from.template_name": templateName, + "status": bson.M{"$ne": setting.ProductStatusDeleting}, + "source": setting.SourceFromChartTemplate, } - return services, nil + return c.listMaxRevisions(m, nil) } -func (c *ServiceColl) ListMaxRevisionsByProduct(productName string) ([]*models.Service, error) { +func (c *ServiceColl) ListMaxRevisionsAllSvcByProduct(productName string) ([]*models.Service, error) { m := bson.M{ "product_name": productName, - "status": bson.M{"$ne": setting.ProductStatusDeleting}, } - return c.listMaxRevisions(m, nil) } @@ -204,15 +218,14 @@ func (c *ServiceColl) Find(opt *ServiceFindOption) (*models.Service, error) { if opt.ServiceName == "" { return nil, fmt.Errorf("ServiceName is empty") } - if opt.ProductName == "" { - return nil, fmt.Errorf("ProductName is empty") - } query := bson.M{} query["service_name"] = opt.ServiceName - query["product_name"] = opt.ProductName - service := new(models.Service) + if opt.ProductName != "" { + query["product_name"] = opt.ProductName + } + service := new(models.Service) if opt.Type != "" { query["type"] = opt.Type } @@ -229,6 +242,9 @@ func (c *ServiceColl) Find(opt *ServiceFindOption) (*models.Service, error) { err := c.FindOne(context.TODO(), query, opts).Decode(service) if err != nil { + if err == mongo.ErrNoDocuments && opt.IgnoreNoDocumentErr { + return nil, nil + } return nil, err } @@ -280,6 +296,27 @@ func (c *ServiceColl) Create(args *models.Service) error { return err } +func (c *ServiceColl) UpdateServiceHealthCheckStatus(args *models.Service) error { + // avoid panic issue + if args == nil { + return errors.New("nil ServiceTmplObject") + } + args.ProductName = strings.TrimSpace(args.ProductName) + args.ServiceName = strings.TrimSpace(args.ServiceName) + + query := bson.M{"product_name": args.ProductName, "service_name": args.ServiceName, "revision": args.Revision} + + changeMap := bson.M{ + "create_by": args.CreateBy, + "create_time": time.Now().Unix(), + "env_configs": args.EnvConfigs, + "env_statuses": args.EnvStatuses, + } + change := bson.M{"$set": changeMap} + _, err := c.UpdateOne(context.TODO(), query, change) + return err +} + func (c *ServiceColl) Update(args *models.Service) error { // avoid panic issue if args == nil { @@ -305,6 +342,75 @@ func (c *ServiceColl) Update(args *models.Service) error { return err } +// ListExternalServicesBy list service only for external services ,other service type not use before refactor +func (c *ServiceColl) ListExternalWorkloadsBy(productName, envName string, serviceNames ...string) ([]*models.Service, error) { + services := make([]*models.Service, 0) + query := bson.M{ + "status": bson.M{"$ne": setting.ProductStatusDeleting}, + } + if productName != "" { + query["product_name"] = productName + } + if envName != "" { + query["env_name"] = envName + } + + if len(serviceNames) > 0 { + query["service_name"] = bson.M{"$in": serviceNames} + } + ctx := context.Background() + cursor, err := c.Collection.Find(ctx, query) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &services) + if err != nil { + return nil, err + } + return services, nil +} + +func (c *ServiceColl) BatchUpdateExternalServicesStatus(productName, envName, status string, serviceNames []string) error { + if productName == "" { + return fmt.Errorf("productName is empty") + } + + if len(serviceNames) == 0 { + return fmt.Errorf("servicenNames is empty") + } + + query := bson.M{"product_name": productName, "service_name": bson.M{"$in": serviceNames}} + if envName != "" { + query["env_name"] = envName + } + + change := bson.M{"$set": bson.M{ + "status": status, + }} + + _, err := c.UpdateMany(context.TODO(), query, change) + return err +} + +// UpdateExternalServiceEnvName only used by external services +func (c *ServiceColl) UpdateExternalServiceEnvName(serviceName, productName, envName string) error { + if serviceName == "" { + return fmt.Errorf("serviceName is empty") + } + if productName == "" { + return fmt.Errorf("productName is empty") + } + + query := bson.M{"service_name": serviceName, "product_name": productName} + change := bson.M{"$set": bson.M{ + "env_name": envName, + }} + + _, err := c.UpdateOne(context.TODO(), query, change) + return err +} + // UpdateExternalServicesStatus only used by external services func (c *ServiceColl) UpdateExternalServicesStatus(serviceName, productName, status, envName string) error { if serviceName == "" && envName == "" { @@ -366,6 +472,95 @@ func (c *ServiceColl) ListAllRevisions() ([]*models.Service, error) { return resp, err } +func (c *ServiceColl) ListServicesWithSRevision(opt *SvcRevisionListOption) ([]*models.Service, error) { + productMatch := bson.M{} + productMatch["product_name"] = opt.ProductName + + var serviceMatch bson.A + for _, sr := range opt.ServiceRevisions { + serviceMatch = append(serviceMatch, bson.M{ + "service_name": sr.ServiceName, + "revision": sr.Revision, + }) + } + + pipeline := []bson.M{ + { + "$match": productMatch, + }, + { + "$match": bson.M{ + "$or": serviceMatch, + }, + }, + } + cursor, err := c.Aggregate(context.TODO(), pipeline) + if err != nil { + return nil, err + } + + res := make([]*models.Service, 0) + if err := cursor.All(context.TODO(), &res); err != nil { + return nil, err + } + return res, err +} + +func (c *ServiceColl) ListMaxRevisionsByProject(serviceName, serviceType string) ([]*models.Service, error) { + + pipeline := []bson.M{ + { + "$match": bson.M{ + "status": bson.M{"$ne": setting.ProductStatusDeleting}, + "service_name": serviceName, + "type": serviceType, + }, + }, + { + "$group": bson.M{ + "_id": bson.M{ + "product_name": "$product_name", + "service_name": "$service_name", + }, + "revision": bson.M{"$max": "$revision"}, + "service_id": bson.M{"$last": "$_id"}, + }, + }, + } + + cursor, err := c.Aggregate(context.TODO(), pipeline) + if err != nil { + return nil, err + } + + res := make([]*ServiceAggregateResult, 0) + if err := cursor.All(context.TODO(), &res); err != nil { + return nil, err + } + + var ids []primitive.ObjectID + for _, service := range res { + ids = append(ids, service.ServiceID) + } + + if len(ids) == 0 { + return nil, nil + } + + var resp []*models.Service + query := bson.M{"_id": bson.M{"$in": ids}} + cursor, err = c.Collection.Find(context.TODO(), query) + if err != nil { + return nil, err + } + err = cursor.All(context.TODO(), &resp) + if err != nil { + return nil, err + } + + return resp, nil +} + func (c *ServiceColl) ListMaxRevisions(opt *ServiceListOption) ([]*models.Service, error) { preMatch := bson.M{"status": bson.M{"$ne": setting.ProductStatusDeleting}} postMatch := bson.M{} @@ -377,9 +572,7 @@ func (c *ServiceColl) ListMaxRevisions(opt *ServiceListOption) ([]*models.Servic if opt.ServiceName != "" { preMatch["service_name"] = opt.ServiceName } - if opt.BuildName != "" { - preMatch["build_name"] = opt.BuildName - } + if opt.Source != "" { preMatch["source"] = opt.Source } @@ -391,6 +584,9 @@ func (c *ServiceColl) ListMaxRevisions(opt *ServiceListOption) ([]*models.Servic } // post options (anything that changes over revision should be added in post options) + if opt.BuildName != "" { + postMatch["build_name"] = opt.BuildName + } if opt.Visibility != "" { postMatch["visibility"] = opt.Visibility } @@ -416,18 +612,22 @@ func (c *ServiceColl) ListMaxRevisions(opt *ServiceListOption) ([]*models.Servic } postMatch["_id"] = bson.M{"$in": srs} } + } return c.listMaxRevisions(preMatch, postMatch) } func (c *ServiceColl) Count(productName string) (int, error) { + match := bson.M{ + "status": bson.M{"$ne": setting.ProductStatusDeleting}, + } + if productName != "" { + match["product_name"] = productName + } pipeline := []bson.M{ { - "$match": bson.M{ - "product_name": productName, - "status": bson.M{"$ne": setting.ProductStatusDeleting}, - }, + "$match": match, }, { "$group": bson.M{ @@ -457,6 +657,15 @@ func (c *ServiceColl) Count(productName string) (int, error) { return cs[0].Count, nil } +func (c *ServiceColl) GetTemplateReference(templateID string) ([]*models.Service, error) { + query := bson.M{ + "template_id": templateID, + "status": bson.M{"$ne": setting.ProductStatusDeleting}, + } + + return c.listMaxRevisions(query, nil) +} + func (c *ServiceColl) listMaxRevisions(preMatch, postMatch bson.M) ([]*models.Service, error) { var pipeResp []*grouped pipeline := []bson.M{ @@ -474,6 +683,7 @@ func (c *ServiceColl) listMaxRevisions(preMatch, postMatch bson.M) ([]*models.Se }, "service_id": bson.M{"$last": "$_id"}, "visibility": bson.M{"$last": "$visibility"}, + "build_name": bson.M{"$last": "$build_name"}, }, }, } diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/services_in_external_env.go b/pkg/microservice/aslan/core/common/repository/mongodb/services_in_external_env.go new file mode 100644 index 0000000000000000000000000000000000000000..182374ee0c8ad6079b043db5d4082ea695ce87f7 --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/mongodb/services_in_external_env.go @@ -0,0 +1,155 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "errors" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type ServicesInExternalEnvColl struct { + *mongo.Collection + + coll string +} + +func NewServicesInExternalEnvColl() *ServicesInExternalEnvColl { + name := models.ServicesInExternalEnv{}.TableName() + return &ServicesInExternalEnvColl{ + Collection: mongotool.Database(config.MongoDatabase()).Collection(name), + coll: name, + } +} + +func (c *ServicesInExternalEnvColl) GetCollectionName() string { + return c.coll +} + +func (c *ServicesInExternalEnvColl) EnsureIndex(ctx context.Context) error { + mods := []mongo.IndexModel{ + { + Keys: bson.D{ + bson.E{Key: "product_name", Value: 1}, + bson.E{Key: "env_name", Value: 1}, + bson.E{Key: "service_name", Value: 1}, + }, + Options: options.Index().SetUnique(true), + }, + { + Keys: bson.D{ + bson.E{Key: "namespace", Value: 1}, + bson.E{Key: "cluster_id", Value: 1}, + }, + Options: options.Index().SetUnique(false), + }, + } + + _, err := c.Indexes().CreateMany(ctx, mods) + + return err +} + +// Create ... +func (c *ServicesInExternalEnvColl) Create(args *models.ServicesInExternalEnv) error { + // avoid panic issue + if args == nil { + return errors.New("nil ServicesInExternalEnv") + } + + _, err := c.InsertOne(context.TODO(), args) + + return err +} + +type ServicesInExternalEnvArgs struct { + ProductName string + ServiceName string + EnvName string + Namespace string + ClusterID string + ExcludeEnvName string +} + +func (c *ServicesInExternalEnvColl) List(args *ServicesInExternalEnvArgs) ([]*models.ServicesInExternalEnv, error) { + if args == nil { + return nil, errors.New("nil ServicesInExternalEnvArgs") + } + query := bson.M{} + if args.ProductName != "" { + query["product_name"] = args.ProductName + } + + if args.EnvName != "" { + query["env_name"] = args.EnvName + } + + if args.ExcludeEnvName != "" { + query["env_name"] = bson.M{"$ne": args.ExcludeEnvName} + } + + if args.Namespace != "" { + query["namespace"] = args.Namespace + } + + if args.ClusterID != "" { + query["cluster_id"] = args.ClusterID + } + + resp := make([]*models.ServicesInExternalEnv, 0) + ctx := context.Background() + + cursor, err := c.Collection.Find(ctx, query) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &resp) + if err != nil { + return nil, err + } + + return resp, err +} + +func (c *ServicesInExternalEnvColl) Delete(args *ServicesInExternalEnvArgs) error { + if args == nil { + return errors.New("nil ServicesInExternalEnvArgs") + } + query := bson.M{} + if args.ProductName != "" { + query["product_name"] = args.ProductName + } + + if args.EnvName != "" { + query["env_name"] = args.EnvName + } + + if args.ServiceName != "" { + query["service_name"] = args.ServiceName + } + + _, err := c.DeleteMany(context.TODO(), query) + return err +} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/settings.go b/pkg/microservice/aslan/core/common/repository/mongodb/settings.go new file mode 100644 index 0000000000000000000000000000000000000000..8b849a89e5b20c3798dc6920b43e0df55fd88be1 --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/mongodb/settings.go @@ -0,0 +1,109 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/setting" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type SystemSettingColl struct { + *mongo.Collection + + coll string +} + +func NewSystemSettingColl() *SystemSettingColl { + name := models.SystemSetting{}.TableName() + return &SystemSettingColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} +} + +func (c *SystemSettingColl) GetCollectionName() string { + return c.coll +} + +func (c *SystemSettingColl) EnsureIndex(ctx context.Context) error { + return nil +} + +func (c *SystemSettingColl) Get() (*models.SystemSetting, error) { + query := bson.M{} + resp := &models.SystemSetting{} + + err := c.FindOne(context.TODO(), query).Decode(resp) + return resp, err +} + +func (c *SystemSettingColl) UpdateDefaultLoginSetting(defaultLogin string) error { + id, _ := primitive.ObjectIDFromHex(setting.LocalClusterID) + change := bson.M{"$set": bson.M{ + "default_login": defaultLogin, + }} + query := bson.M{"_id": id} + _, err := c.UpdateOne(context.TODO(), query, change) + return err +} + +func (c *SystemSettingColl) UpdateConcurrencySetting(workflowConcurrency, buildConcurrency int64) error { + id, _ := primitive.ObjectIDFromHex(setting.LocalClusterID) + change := bson.M{"$set": bson.M{ + "workflow_concurrency": workflowConcurrency, + "build_concurrency": buildConcurrency, + }} + query := bson.M{"_id": id} + _, err := c.UpdateOne(context.TODO(), query, change) + return err +} + +func (c *SystemSettingColl) InitSystemSettings() error { + _, err := c.Get() + // if we didn't find anything + if err != nil { + return c.CreateOrUpdate(setting.LocalClusterID, &models.SystemSetting{ + WorkflowConcurrency: 2, + BuildConcurrency: 5, + DefaultLogin: setting.DefaultLoginLocal, + }) + } + return nil +} + +func (c *SystemSettingColl) CreateOrUpdate(id string, args *models.SystemSetting) error { + var objectID primitive.ObjectID + if id != "" { + objectID, _ = primitive.ObjectIDFromHex(id) + } else { + objectID = primitive.NewObjectID() + } + + args.UpdateTime = time.Now().Unix() + + query := bson.M{"_id": objectID} + change := bson.M{"$set": args} + _, err := c.UpdateOne(context.TODO(), query, change, options.Update().SetUpsert(true)) + return err +} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/sonar.go b/pkg/microservice/aslan/core/common/repository/mongodb/sonar.go new file mode 100644 index 0000000000000000000000000000000000000000..f0f3e0dae62dc5c0e0dfa1782046c6d500d80589 --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/mongodb/sonar.go @@ -0,0 +1,132 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "errors" + "fmt" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type SonarIntegrationColl struct { + *mongo.Collection + + coll string +} + +func NewSonarIntegrationColl() *SonarIntegrationColl { + name := models.SonarIntegration{}.TableName() + return &SonarIntegrationColl{ + Collection: mongotool.Database(config.MongoDatabase()).Collection(name), + coll: name, + } +} + +func (c *SonarIntegrationColl) GetCollectionName() string { + return c.coll +} + +func (c *SonarIntegrationColl) EnsureIndex(ctx context.Context) error { + return nil +} + +func (c *SonarIntegrationColl) Create(ctx context.Context, args *models.SonarIntegration) error { + if args == nil { + return errors.New("sonar integration is nil") + } + + _, err := c.InsertOne(ctx, args) + + return err +} + +func (c *SonarIntegrationColl) List(ctx context.Context, pageNum, pageSize int64) ([]*models.SonarIntegration, int64, error) { + query := bson.M{} + resp := make([]*models.SonarIntegration, 0) + + opt := options.Find() + + if pageNum != 0 && pageSize != 0 { + opt = opt.SetSkip((pageNum - 1) * pageSize).SetLimit(pageSize) + } + + cursor, err := c.Collection.Find(ctx, query, opt) + if err != nil { + return nil, 0, err + } + + err = cursor.All(ctx, &resp) + if err != nil { + return nil, 0, err + } + count, err := c.Collection.CountDocuments(ctx, query) + if err != nil { + return nil, 0, err + } + + return resp, count, nil +} + +func (c *SonarIntegrationColl) GetByID(ctx context.Context, idstring string) (*models.SonarIntegration, error) { + resp := new(models.SonarIntegration) + id, err := primitive.ObjectIDFromHex(idstring) + if err != nil { + return nil, err + } + query := bson.M{"_id": id} + + err = c.FindOne(ctx, query).Decode(&resp) + if err != nil { + return nil, err + } + return resp, nil +} + +func (c *SonarIntegrationColl) Update(ctx context.Context, idString string, obj *models.SonarIntegration) error { + if obj == nil { + return fmt.Errorf("nil object") + } + id, err := primitive.ObjectIDFromHex(idString) + if err != nil { + return fmt.Errorf("invalid id") + } + filter := bson.M{"_id": id} + update := bson.M{"$set": obj} + + _, err = c.UpdateOne(ctx, filter, update) + return err +} + +func (c *SonarIntegrationColl) DeleteByID(ctx context.Context, idstring string) error { + id, err := primitive.ObjectIDFromHex(idstring) + if err != nil { + return err + } + query := bson.M{"_id": id} + + _, err = c.DeleteOne(ctx, query) + return err +} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/task.go b/pkg/microservice/aslan/core/common/repository/mongodb/task.go index 2640d1ccaea18e209c01ac65a5c59fcde0369d66..ec94620e694330adc85437b0b60e33b1c2fae8d7 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/task.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/task.go @@ -30,23 +30,30 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" mongotool "github.com/koderover/zadig/pkg/tool/mongo" + "github.com/koderover/zadig/pkg/types" ) type ListTaskOption struct { - PipelineName string - Status config.Status - Team string - TeamID int - Limit int - Skip int - Detail bool - Type config.PipelineType - CreateTime int64 - TaskCreator string - Source string - MergeRequestID string - NeedTriggerBy bool - NeedAllData bool + PipelineName string + PipelineNames []string + Status config.Status + Statuses []string + Team string + TeamID int + Limit int + Skip int + Detail bool + ForWorkflowTaskList bool + Type config.PipelineType + CreateTime int64 + TaskCreator string + TaskCreators []string + Source string + Committers []string + ServiceModule []string + MergeRequestID string + NeedTriggerBy bool + NeedAllData bool } type FindTaskOption struct { @@ -55,26 +62,41 @@ type FindTaskOption struct { Type config.PipelineType } +type CodeInfo struct { + AuthorName string `json:"author_name"` + CommitId string `json:"commit_id"` + CommitMessage string `json:"commit_message"` +} + +type ServiceModule struct { + ServiceModule string `json:"service_module"` + ServiceName string `json:"service_name"` + CodeInfo []*types.Repository `json:"code_info"` +} + +type BuildStage struct { + SubTasks map[string]task.Build `bson:"sub_tasks" json:"-"` +} + type TaskPreview struct { - TaskID int64 `bson:"task_id" json:"task_id"` - TaskCreator string `bson:"task_creator" json:"task_creator"` - ProductName string `bson:"product_name" json:"product_name"` - PipelineName string `bson:"pipeline_name" json:"pipeline_name"` - Namespace string `bson:"namespace" json:"namespace"` - ServiceName string `bson:"service_name" json:"service_name"` - Status config.Status `bson:"status" json:"status"` - CreateTime int64 `bson:"create_time" json:"create_time,omitempty"` - StartTime int64 `bson:"start_time" json:"start_time,omitempty"` - EndTime int64 `bson:"end_time" json:"end_time,omitempty"` - SubTasks []*map[string]interface{} `bson:"sub_tasks,omitempty" json:"sub_tasks,omitempty"` - TaskArgs *models.TaskArgs `bson:"task_args" json:"task_args"` - WorkflowArgs *models.WorkflowTaskArgs `bson:"workflow_args" json:"workflow_args"` - TestReports map[string]interface{} `bson:"test_reports,omitempty" json:"test_reports,omitempty"` - Type config.PipelineType `bson:"type" json:"type"` - Stages []*models.Stage `bson:"stages" json:"stages,omitempty"` - // 服务名称，用于任务列表的展示 - BuildServices []string `bson:"-" json:"build_services"` - TriggerBy *models.TriggerBy `bson:"trigger_by,omitempty" json:"trigger_by,omitempty"` + TaskID int64 `bson:"task_id" json:"task_id"` + TaskCreator string `bson:"task_creator" json:"task_creator"` + ProductName string `bson:"product_name" json:"product_name"` + PipelineName string `bson:"pipeline_name" json:"pipeline_name"` + Namespace string `bson:"namespace" json:"namespace"` + ServiceName string `bson:"service_name" json:"service_name"` + Status config.Status `bson:"status" json:"status"` + CreateTime int64 `bson:"create_time" json:"create_time,omitempty"` + StartTime int64 `bson:"start_time" json:"start_time,omitempty"` + EndTime int64 `bson:"end_time" json:"end_time,omitempty"` + SubTasks []*map[string]interface{} `bson:"sub_tasks,omitempty" json:"sub_tasks,omitempty"` + TaskArgs *models.TaskArgs `bson:"task_args" json:"task_args"` + WorkflowArgs *models.WorkflowTaskArgs `bson:"workflow_args" json:"workflow_args"` + TestReports map[string]interface{} `bson:"test_reports,omitempty" json:"test_reports,omitempty"` + Type config.PipelineType `bson:"type" json:"type"` + Stages []*models.Stage `bson:"stages" json:"stages,omitempty"` + ServiceModules []*ServiceModule `bson:"-" json:"service_modules"` + TriggerBy *models.TriggerBy `bson:"trigger_by,omitempty" json:"trigger_by,omitempty"` } type ListAllTaskOption struct { @@ -125,6 +147,14 @@ func (c *TaskColl) EnsureIndex(ctx context.Context) error { }, Options: options.Index().SetUnique(false), }, + { + Keys: bson.D{ + bson.E{Key: "pipeline_name", Value: 1}, + bson.E{Key: "is_archived", Value: 1}, + bson.E{Key: "is_deleted", Value: 1}, + }, + Options: options.Index().SetUnique(false).SetBackground(true), + }, } _, err := c.Indexes().CreateMany(ctx, mod) @@ -191,6 +221,63 @@ func (c *TaskColl) FindLatestTask(args *FindTaskOption) (*task.Task, error) { return res, err } +type TaskInfo struct { + ID taskGrouped `bson:"_id"` + TaskID int64 `bson:"task_id"` + PipelineName string `bson:"pipeline_name"` + Status string `bson:"status"` +} + +type taskGrouped struct { + PipelineName string `bson:"pipeline_name"` + Type config.PipelineType `bson:"type"` +} + +func (c *TaskColl) ListRecentTasks(args *ListTaskOption) ([]*TaskInfo, error) { + query := bson.M{"is_deleted": false} + if args.Type != "" { + query["type"] = args.Type + } + if args.Status != "" { + query["status"] = args.Status + } + if len(args.PipelineNames) > 0 { + query["pipeline_name"] = bson.M{"$in": args.PipelineNames} + } + + pipeline := []bson.M{ + { + "$match": query, + }, + { + "$sort": bson.M{"task_id": -1}, + }, + { + "$group": bson.M{ + "_id": bson.D{ + {"pipeline_name", "$pipeline_name"}, + {"type", "$type"}, + }, + "pipeline_name": bson.M{"$first": "$pipeline_name"}, + "task_id": bson.M{"$first": "$task_id"}, + "status": bson.M{"$first": "$status"}, + }, + }, + } + + cursor, err := c.Aggregate(context.TODO(), pipeline) + if err != nil { + return nil, err + } + + res := make([]*TaskInfo, 0) + if err := cursor.All(context.TODO(), &res); err != nil { + return nil, err + } + + return res, nil +} + func (c *TaskColl) List(option *ListTaskOption) (ret []*TaskPreview, err error) { ret = make([]*TaskPreview, 0) if option == nil { @@ -207,21 +294,32 @@ func (c *TaskColl) List(option *ListTaskOption) (ret []*TaskPreview, err error) if option.PipelineName != "" { query["pipeline_name"] = option.PipelineName } - if option.Status != "" { + if len(option.Statuses) > 0 { + query["status"] = bson.M{"$in": option.Statuses} + } else if option.Status != "" { query["status"] = option.Status } if option.Team != "" { query["team"] = option.Team } - if option.TaskCreator != "" { + if len(option.TaskCreators) > 0 { + query["task_creator"] = bson.M{"$in": option.TaskCreators} + } else if option.TaskCreator != "" { query["task_creator"] = option.TaskCreator } + if len(option.Committers) > 0 { + query["workflow_args.committer"] = bson.M{"$in": option.Committers} + } + if len(option.ServiceModule) > 0 { + query["workflow_args.targets.name"] = bson.M{"$in": option.ServiceModule} + } if option.Source != "" { query["trigger_by.source"] = option.Source } if option.MergeRequestID != "" { query["trigger_by.merge_request_id"] = option.MergeRequestID } + query["is_archived"] = false query["is_deleted"] = false //是否需要subtask信息 @@ -256,8 +354,19 @@ func (c *TaskColl) List(option *ListTaskOption) (ret []*TaskPreview, err error) selector = append(selector, bson.E{"test_reports", 1}) selector = append(selector, bson.E{"stages", 1}) } + + // show necessary fields when listing workflow tasks + if option.ForWorkflowTaskList { + selector = append(selector, bson.E{"workflow_args.targets.name", 1}) + selector = append(selector, bson.E{"workflow_args.targets.service_name", 1}) + selector = append(selector, bson.E{"workflow_args.namespace", 1}) + selector = append(selector, bson.E{"test_reports", 1}) + selector = append(selector, bson.E{"stages", 1}) + } + opt := options.Find() opt.SetSort(bson.D{{"create_time", -1}}) + if option.Limit != 0 { opt.SetProjection(selector).SetSkip(int64(option.Skip)).SetLimit(int64(option.Limit)) } else if option.CreateTime > 0 { @@ -271,7 +380,39 @@ func (c *TaskColl) List(option *ListTaskOption) (ret []*TaskPreview, err error) cursor, err := c.Collection.Find(context.TODO(), query, opt) if err != nil { - return + return ret, err + } + + if err = cursor.All(context.TODO(), &ret); err != nil { + return nil, err + } + return +} + +func (c *TaskColl) ListPreview(pipelineNames []string) (ret []*TaskPreview, err error) { + ret = make([]*TaskPreview, 0) + query := bson.M{} + if pipelineNames != nil { + query["pipeline_name"] = bson.M{"$in": pipelineNames} + } + query["is_archived"] = false + query["is_deleted"] = false + selector := bson.D{ + {"task_id", 1}, + {"task_creator", 1}, + {"product_name", 1}, + {"pipeline_name", 1}, + {"status", 1}, + {"create_time", 1}, + {"start_time", 1}, + {"end_time", 1}, + {"type", 1}, + } + opt := options.Find() + opt.SetProjection(selector) + cursor, err := c.Collection.Find(context.TODO(), query, opt) + if err != nil { + return ret, err } err = cursor.All(context.TODO(), &ret) return @@ -332,7 +473,11 @@ func (c *TaskColl) Create(args *task.Task) error { type CountTaskOption struct { PipelineNames []string + Statuses []string Status config.TaskStatus + TaskCreators []string + Committers []string + ServiceModule []string Type config.PipelineType } @@ -342,15 +487,27 @@ func (c *TaskColl) Count(option *CountTaskOption) (ret int, err error) { } query := bson.M{} + if len(option.TaskCreators) > 0 { + query["task_creator"] = bson.M{"$in": option.TaskCreators} + } + if len(option.Committers) > 0 { + query["workflow_args.committer"] = bson.M{"$in": option.Committers} + } + if len(option.ServiceModule) > 0 { + query["workflow_args.targets.name"] = bson.M{"$in": option.ServiceModule} + } if option.PipelineNames != nil { query["pipeline_name"] = bson.M{"$in": option.PipelineNames} } - if option.Status != "" { + if len(option.Statuses) > 0 { + query["status"] = bson.M{"$in": option.Statuses} + } else if option.Status != "" { query["status"] = option.Status } if option.Type != "" { query["type"] = option.Type } + query["is_archived"] = false query["is_deleted"] = false // 仅支持查询最近三个月的数据 @@ -519,13 +676,21 @@ func (c *TaskColl) UpdateUnfinishedTask(args *task.Task) error { return err } -func (c *TaskColl) ArchiveHistoryPipelineTask(pipelineName string, taskType config.PipelineType, remain int) error { +func (c *TaskColl) ArchiveHistoryPipelineTask(pipelineName string, taskType config.PipelineType, remain, remainDays int) error { + if remain == 0 && remainDays == 0 { + return nil + } query := bson.M{"pipeline_name": pipelineName, "type": taskType, "is_deleted": false} count, err := c.CountDocuments(context.TODO(), query) if err != nil { return err } - query["task_id"] = bson.M{"$lt": int(count) - remain + 1} + if remain > 0 { + query["task_id"] = bson.M{"$lt": int(count) - remain + 1} + } + if remainDays > 0 { + query["create_time"] = bson.M{"$lt": time.Now().AddDate(0, 0, -remainDays).Unix()} + } change := bson.M{"$set": bson.M{ "is_archived": true, }} @@ -533,3 +698,13 @@ func (c *TaskColl) ArchiveHistoryPipelineTask(pipelineName string, taskType conf return err } + +func (c *TaskColl) DistinctFieldsPipelineTask(fieldName, projectName, pipelineName string, typeString config.PipelineType, isNeedAllData bool) ([]interface{}, error) { + query := bson.M{"product_name": projectName, "pipeline_name": pipelineName, "type": typeString, "is_deleted": false} + if isNeedAllData { + createTime := timeutil.BeginningOfDay().AddDate(0, -3, 0).Unix() + query["create_time"] = bson.M{"$gt": createTime} + } + res, err := c.Distinct(context.TODO(), fieldName, query) + return res, err +} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/template/product.go b/pkg/microservice/aslan/core/common/repository/mongodb/template/product.go index 430da12583376aabf10a366879f62d6514b9ff84..0b09255ef840d3e3565be58b990847e9027e7297 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/template/product.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/template/product.go @@ -31,6 +31,19 @@ import ( mongotool "github.com/koderover/zadig/pkg/tool/mongo" ) +type ProjectInfo struct { + Name string `bson:"product_name"` + Alias string `bson:"project_name"` + Desc string `bson:"description"` + UpdatedAt int64 `bson:"update_time"` + UpdatedBy string `bson:"update_by"` + OnboardStatus int `bson:"onboarding_status"` + Public bool `bson:"public"` + DeployType string `bson:"deploy_type"` + CreateEnvType string `bson:"create_env_type"` + BasicFacility string `bson:"basic_facility"` +} + type ProductColl struct { *mongo.Collection @@ -71,53 +84,97 @@ func (c *ProductColl) FindProjectName(project string) (*template.Product, error) return resp, err } -func (c *ProductColl) List() ([]*template.Product, error) { - var resp []*template.Product - - cursor, err := c.Collection.Find(context.TODO(), bson.M{}) +func (c *ProductColl) ListNames(inNames []string) ([]string, error) { + res, err := c.listProjects(inNames, bson.M{ + "product_name": "$product_name", + }) if err != nil { return nil, err } - err = cursor.All(context.TODO(), &resp) - if err != nil { - return nil, err + + var names []string + for _, r := range res { + names = append(names, r.Name) } - return resp, nil + return names, nil } -func (c *ProductColl) ListNames() ([]string, error) { - var res []struct { - ProductName string `bson:"product_name"` +func (c *ProductColl) ListProjectBriefs(inNames []string) ([]*ProjectInfo, error) { + return c.listProjects(inNames, bson.M{ + "product_name": "$product_name", + "project_name": "$project_name", + "description": "$description", + "update_time": "$update_time", + "update_by": "$update_by", + "onboarding_status": "$onboarding_status", + "public": "$public", + "deploy_type": "$product_feature.deploy_type", + "create_env_type": "$product_feature.create_env_type", + "basic_facility": "$product_feature.basic_facility", + }) +} + +func (c *ProductColl) listProjects(inNames []string, projection bson.M) ([]*ProjectInfo, error) { + filter := bson.M{} + if len(inNames) > 0 { + filter["product_name"] = bson.M{"$in": inNames} } - opts := options.Find() - projection := bson.D{ - {"product_name", 1}, + pipeline := []bson.M{ + { + "$match": filter, + }, + { + "$project": projection, + }, } - opts.SetProjection(projection) - cursor, err := c.Collection.Find(context.TODO(), bson.M{}, opts) + cursor, err := c.Collection.Aggregate(context.TODO(), pipeline) if err != nil { return nil, err } + + var res []*ProjectInfo err = cursor.All(context.TODO(), &res) if err != nil { return nil, err } + return res, nil +} - var names []string - for _, r := range res { - names = append(names, r.ProductName) +func (c *ProductColl) Count() (int64, error) { + query := bson.M{} + + ctx := context.Background() + count, err := c.Collection.CountDocuments(ctx, query) + if err != nil { + return 0, err } - return names, nil + return count, nil +} + +func (c *ProductColl) List() ([]*template.Product, error) { + var resp []*template.Product + + cursor, err := c.Collection.Find(context.TODO(), bson.M{}) + if err != nil { + return nil, err + } + err = cursor.All(context.TODO(), &resp) + if err != nil { + return nil, err + } + + return resp, nil } type ProductListOpt struct { IsOpensource string ContainSharedServices []*template.ServiceInfo BasicFacility string + DeployType string } // ListWithOption ... @@ -134,6 +191,9 @@ func (c *ProductColl) ListWithOption(opt *ProductListOpt) ([]*template.Product, if opt.BasicFacility != "" { query["product_feature.basic_facility"] = opt.BasicFacility } + if opt.DeployType != "" { + query["product_feature.deploy_type"] = bson.M{"$in": strings.Split(opt.DeployType, ",")} + } cursor, err := c.Collection.Find(context.TODO(), query) if err != nil { @@ -182,6 +242,19 @@ func (c *ProductColl) Create(args *template.Product) error { return err } +func (c *ProductColl) UpdateServiceOrchestration(productName string, services [][]string, updateBy string) error { + + query := bson.M{"product_name": productName} + change := bson.M{"$set": bson.M{ + "services": services, + "update_time": time.Now().Unix(), + "update_by": updateBy, + }} + + _, err := c.UpdateOne(context.TODO(), query, change) + return err +} + // Update existing ProductTmpl func (c *ProductColl) Update(productName string, args *template.Product) error { // avoid panic issue @@ -198,31 +271,43 @@ func (c *ProductColl) Update(productName string, args *template.Product) error { "services": args.Services, "update_time": time.Now().Unix(), "update_by": args.UpdateBy, - "teams": args.Teams, "enabled": args.Enabled, "description": args.Description, - "visibility": args.Visibility, - "user_ids": args.UserIDs, - "team_id": args.TeamID, "timeout": args.Timeout, + "auto_deploy": args.AutoDeploy, "shared_services": args.SharedServices, "image_searching_rules": args.ImageSearchingRules, "custom_tar_rule": args.CustomTarRule, "custom_image_rule": args.CustomImageRule, + "delivery_version_hook": args.DeliveryVersionHook, + "public": args.Public, }} _, err := c.UpdateOne(context.TODO(), query, change) return err } +type ProductArgs struct { + ProductName string `json:"product_name"` + Services [][]string `json:"services"` + UpdateBy string `json:"update_by"` +} + // AddService adds a service to services[0] if it is not there. func (c *ProductColl) AddService(productName, serviceName string) error { query := bson.M{"product_name": productName} + serviceUniqueFilter := bson.M{ + "$elemMatch": bson.M{ + "$elemMatch": bson.M{ + "$eq": serviceName, + }, + }, + } + query["services"] = bson.M{"$not": serviceUniqueFilter} change := bson.M{"$addToSet": bson.M{ - "services.0": serviceName, + "services.1": serviceName, }} - _, err := c.UpdateOne(context.TODO(), query, change) return err } diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/webhook.go b/pkg/microservice/aslan/core/common/repository/mongodb/webhook.go index 39f153ba1955014c7f254501f7ee867b9e8913b3..67bb6060348701f9a54d660bf147fe0dde50e804 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/webhook.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/webhook.go @@ -119,6 +119,20 @@ func (c *WebHookColl) Delete(owner, repo, address string) error { return nil } +func (c *WebHookColl) List() ([]*models.WebHook, error) { + res := make([]*models.WebHook, 0) + cursor, err := c.Collection.Find(context.TODO(), bson.M{}) + if err != nil { + return nil, err + } + err = cursor.All(context.TODO(), &res) + if err != nil { + return nil, err + } + + return res, err +} + // Find find webhook func (c *WebHookColl) Find(owner, repo, address string) (*models.WebHook, error) { query := bson.M{"owner": owner, "repo": repo, "address": address} @@ -137,3 +151,7 @@ func (c *WebHookColl) Update(owner, repo, address, hookID string) error { _, err := c.UpdateOne(context.TODO(), query, change) return err } + +func IsErrNoDocuments(err error) bool { + return err == mongo.ErrNoDocuments +} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/workflow.go b/pkg/microservice/aslan/core/common/repository/mongodb/workflow.go index 4f482802bbac62412fbaddeb176a54d8e5ddc84e..d2c608052a230c17e2e453005021f7bac355d6e6 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/workflow.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/workflow.go @@ -22,6 +22,7 @@ import ( "time" "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" @@ -31,9 +32,46 @@ import ( ) type ListWorkflowOption struct { - ProductName string - IsSort bool - TestName string + IsSort bool + Projects []string + Names []string + Ids []string +} + +type Workflow struct { + Name string `json:"name"` + ProjectName string `json:"projectName"` +} + +type ListWorkflowOpt struct { + Workflows []Workflow +} + +func (c *WorkflowColl) ListByWorkflows(opt ListWorkflowOpt) ([]*models.Workflow, error) { + var res []*models.Workflow + + if len(opt.Workflows) == 0 { + return nil, nil + } + condition := bson.A{} + for _, workflow := range opt.Workflows { + condition = append(condition, bson.M{ + "name": workflow.Name, + "product_tmpl_name": workflow.ProjectName, + }) + } + filter := bson.D{{"$or", condition}} + cursor, err := c.Collection.Find(context.TODO(), filter) + if err == mongo.ErrNoDocuments { + return nil, nil + } + if err != nil { + return nil, err + } + if err := cursor.All(context.TODO(), &res); err != nil { + return nil, err + } + return res, nil } type WorkflowColl struct { @@ -64,8 +102,22 @@ func (c *WorkflowColl) EnsureIndex(ctx context.Context) error { func (c *WorkflowColl) List(opt *ListWorkflowOption) ([]*models.Workflow, error) { query := bson.M{} - if opt.ProductName != "" { - query["product_tmpl_name"] = opt.ProductName + if len(opt.Projects) > 0 { + query["product_tmpl_name"] = bson.M{"$in": opt.Projects} + } + if len(opt.Names) > 0 { + query["name"] = bson.M{"$in": opt.Names} + } + if len(opt.Ids) > 0 { + var oids []primitive.ObjectID + for _, id := range opt.Ids { + oid, err := primitive.ObjectIDFromHex(id) + if err != nil { + return nil, err + } + oids = append(oids, oid) + } + query["_id"] = bson.M{"$in": oids} } resp := make([]*models.Workflow, 0) @@ -87,6 +139,18 @@ func (c *WorkflowColl) List(opt *ListWorkflowOption) ([]*models.Workflow, error) return resp, nil } +func (c *WorkflowColl) Count() (int64, error) { + query := bson.M{} + + ctx := context.Background() + count, err := c.Collection.CountDocuments(ctx, query) + if err != nil { + return 0, err + } + + return count, nil +} + func (c *WorkflowColl) ListByTestName(testName string) ([]*models.Workflow, error) { // 查询test_stage.tests中存在此testName 或者 test_stage.test_names存在此testName的工作流 match := []bson.M{ @@ -143,6 +207,21 @@ func (c *WorkflowColl) Create(args *models.Workflow) error { return err } +func (c *WorkflowColl) BulkCreate(args []*models.Workflow) error { + if len(args) == 0 { + return nil + } + var ois []interface{} + for _, arg := range args { + arg.CreateTime = time.Now().Unix() + arg.UpdateTime = time.Now().Unix() + ois = append(ois, arg) + } + + _, err := c.InsertMany(context.TODO(), ois) + return err +} + func (c *WorkflowColl) Replace(args *models.Workflow) error { if args == nil { return errors.New("nil Workflow args") @@ -172,3 +251,25 @@ func (c *WorkflowColl) ListWithScheduleEnabled() ([]*models.Workflow, error) { return resp, nil } + +func (c *WorkflowColl) ListWorkflowsByProjects(projects []string) ([]*models.Workflow, error) { + resp := make([]*models.Workflow, 0) + query := bson.M{} + if len(projects) != 0 { + query = bson.M{"product_tmpl_name": bson.M{ + "$in": projects, + }} + } + + cursor, err := c.Collection.Find(context.TODO(), query) + if err != nil { + return nil, err + } + + err = cursor.All(context.TODO(), &resp) + if err != nil { + return nil, err + } + + return resp, nil +} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/workflow_queue.go b/pkg/microservice/aslan/core/common/repository/mongodb/workflow_queue.go new file mode 100644 index 0000000000000000000000000000000000000000..4c9a1269888af746401a28b70bae803d6b9f1322 --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/mongodb/workflow_queue.go @@ -0,0 +1,126 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "errors" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type ListWorfklowQueueOption struct { + WorkflowName string + Status config.Status +} + +type WorkflowQueueColl struct { + *mongo.Collection + + coll string +} + +func NewWorkflowQueueColl() *WorkflowQueueColl { + name := models.WorkflowQueue{}.TableName() + return &WorkflowQueueColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} +} + +func (c *WorkflowQueueColl) GetCollectionName() string { + return c.coll +} + +func (c *WorkflowQueueColl) EnsureIndex(ctx context.Context) error { + mod := mongo.IndexModel{ + Keys: bson.D{ + bson.E{Key: "task_id", Value: 1}, + bson.E{Key: "workflow_name", Value: 1}, + bson.E{Key: "create_time", Value: 1}, + }, + Options: options.Index().SetUnique(true), + } + + _, err := c.Indexes().CreateOne(ctx, mod) + + return err +} + +func (c *WorkflowQueueColl) List(opt *ListWorfklowQueueOption) ([]*models.WorkflowQueue, error) { + query := bson.M{} + if opt != nil { + if opt.WorkflowName != "" { + query["workflow_name"] = opt.WorkflowName + } + if opt.Status != "" { + query["status"] = opt.Status + } + } + + var resp []*models.WorkflowQueue + ctx := context.Background() + opts := options.Find().SetSort(bson.D{{"create_time", 1}}) + cursor, err := c.Collection.Find(ctx, query, opts) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &resp) + if err != nil { + return nil, err + } + + return resp, nil +} + +func (c *WorkflowQueueColl) Delete(args *models.WorkflowQueue) error { + if args == nil { + return errors.New("nil workflow queue") + } + + query := bson.M{"task_id": args.TaskID, "workflow_name": args.WorkflowName, "create_time": args.CreateTime} + _, err := c.DeleteOne(context.TODO(), query) + return err +} + +func (c *WorkflowQueueColl) Create(args *models.WorkflowQueue) error { + if args == nil { + return errors.New("nil workflow queue") + } + + _, err := c.InsertOne(context.TODO(), args) + return err +} + +func (c *WorkflowQueueColl) Update(args *models.WorkflowQueue) error { + if args == nil { + return errors.New("nil workflow queue") + } + + query := bson.M{"task_id": args.TaskID, "workflow_name": args.WorkflowName, "create_time": args.CreateTime} + change := bson.M{"$set": bson.M{ + "status": args.Status, + "stages": args.Stages, + }} + + _, err := c.UpdateOne(context.TODO(), query, change) + return err +} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/workflow_stat.go b/pkg/microservice/aslan/core/common/repository/mongodb/workflow_stat.go index d0d65ee859170537a011a39e424cb16a434c4814..29133d356bb3ef690dc68fa2c9ed7f34e9ffe216 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/workflow_stat.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/workflow_stat.go @@ -93,8 +93,8 @@ func (c *WorkflowStatColl) Delete(name, workflowType string) error { } type WorkflowStatArgs struct { - Name string `json:"name"` - Type string `json:"type"` + Names []string + Type string } func (c *WorkflowStatColl) FindWorkflowStat(args *WorkflowStatArgs) ([]*models.WorkflowStat, error) { @@ -104,13 +104,12 @@ func (c *WorkflowStatColl) FindWorkflowStat(args *WorkflowStatArgs) ([]*models.W } query := bson.M{} - if args.Name != "" { - query["name"] = args.Name - } - if args.Type != "" { query["type"] = args.Type } + if len(args.Names) > 0 { + query["name"] = bson.M{"$in": args.Names} + } ctx := context.Background() diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/workflow_task_v4.go b/pkg/microservice/aslan/core/common/repository/mongodb/workflow_task_v4.go new file mode 100644 index 0000000000000000000000000000000000000000..80a5c0ed7b5c6d63e8bddeb2e8181d504dfd7624 --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/mongodb/workflow_task_v4.go @@ -0,0 +1,237 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +type ListWorkflowTaskV4Option struct { + WorkflowName string + WorkflowNames []string + CreateTime int64 + BeforeCreatTime bool + Limit int + Skip int +} + +type WorkflowTaskv4Coll struct { + *mongo.Collection + + coll string +} + +func NewworkflowTaskv4Coll() *WorkflowTaskv4Coll { + name := models.WorkflowTask{}.TableName() + return &WorkflowTaskv4Coll{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} +} + +func (c *WorkflowTaskv4Coll) GetCollectionName() string { + return c.coll +} + +func (c *WorkflowTaskv4Coll) EnsureIndex(ctx context.Context) error { + mod := []mongo.IndexModel{ + { + Keys: bson.D{ + bson.E{Key: "task_id", Value: 1}, + bson.E{Key: "workflow_name", Value: 1}, + bson.E{Key: "is_deleted", Value: 1}, + bson.E{Key: "status", Value: 1}, + }, + Options: options.Index().SetUnique(false), + }, + { + Keys: bson.M{"create_time": 1}, + Options: options.Index().SetUnique(false), + }, + { + Keys: bson.D{ + bson.E{Key: "workflow_name", Value: 1}, + bson.E{Key: "is_archived", Value: 1}, + bson.E{Key: "is_deleted", Value: 1}, + }, + Options: options.Index().SetUnique(false), + }, + } + + _, err := c.Indexes().CreateMany(ctx, mod) + + return err +} + +func (c *WorkflowTaskv4Coll) Create(obj *models.WorkflowTask) (string, error) { + if obj == nil { + return "", fmt.Errorf("nil object") + } + + res, err := c.InsertOne(context.TODO(), obj) + if err != nil { + return "", err + } + ID, ok := res.InsertedID.(primitive.ObjectID) + if !ok { + return "", errors.New("failed to get object id from create") + } + return ID.Hex(), err +} + +func (c *WorkflowTaskv4Coll) List(opt *ListWorkflowTaskV4Option) ([]*models.WorkflowTask, int64, error) { + resp := make([]*models.WorkflowTask, 0) + query := bson.M{} + if opt.WorkflowName != "" { + query["workflow_name"] = opt.WorkflowName + } + if opt.WorkflowNames != nil { + query["workflow_name"] = bson.M{"$in": opt.WorkflowNames} + } + query["is_archived"] = false + query["is_deleted"] = false + if opt.CreateTime > 0 { + comparison := "$gte" + if opt.BeforeCreatTime { + comparison = "$lte" + } + query["create_time"] = bson.M{comparison: opt.CreateTime} + } + count, err := c.CountDocuments(context.TODO(), query) + if err != nil { + return nil, 0, err + } + + findOption := options.Find() + if opt.Limit > 0 { + findOption.SetSort(bson.D{{"create_time", -1}}) + findOption.SetSkip(int64(opt.Skip)) + findOption.SetLimit(int64(opt.Limit)) + } + + cursor, err := c.Collection.Find(context.TODO(), query, findOption) + if err != nil { + return nil, 0, err + } + err = cursor.All(context.TODO(), &resp) + if err != nil { + return nil, 0, err + } + return resp, count, nil +} + +func (c *WorkflowTaskv4Coll) InCompletedTasks() ([]*models.WorkflowTask, error) { + ret := make([]*models.WorkflowTask, 0) + query := bson.M{"status": bson.M{"$in": []string{"created", "running"}}} + query["is_deleted"] = false + + opt := options.Find() + opt.SetSort(bson.D{{"create_time", 1}}) + + cursor, err := c.Collection.Find(context.TODO(), query, opt) + if err != nil { + return nil, err + } + err = cursor.All(context.TODO(), &ret) + if err != nil { + return nil, err + } + return ret, nil +} + +func (c *WorkflowTaskv4Coll) Find(workflowName string, taskID int64) (*models.WorkflowTask, error) { + resp := new(models.WorkflowTask) + query := bson.M{"workflow_name": workflowName, "task_id": taskID} + + err := c.FindOne(context.TODO(), query).Decode(&resp) + if err != nil { + return nil, err + } + return resp, nil +} + +func (c *WorkflowTaskv4Coll) GetByID(idstring string) (*models.WorkflowTask, error) { + resp := new(models.WorkflowTask) + id, err := primitive.ObjectIDFromHex(idstring) + if err != nil { + return nil, err + } + query := bson.M{"_id": id} + + err = c.FindOne(context.TODO(), query).Decode(&resp) + if err != nil { + return nil, err + } + return resp, nil +} + +func (c *WorkflowTaskv4Coll) Update(idString string, obj *models.WorkflowTask) error { + if obj == nil { + return fmt.Errorf("nil object") + } + id, err := primitive.ObjectIDFromHex(idString) + if err != nil { + return fmt.Errorf("invalid id") + } + filter := bson.M{"_id": id} + update := bson.M{"$set": obj} + + _, err = c.UpdateOne(context.TODO(), filter, update) + return err +} + +func (c *WorkflowTaskv4Coll) DeleteByWorkflowName(workflowName string) error { + query := bson.M{"workflow_name": workflowName} + change := bson.M{"$set": bson.M{ + "is_deleted": true, + "is_archived": true, + }} + + _, err := c.UpdateMany(context.TODO(), query, change) + return err +} + +func (c *WorkflowTaskv4Coll) ArchiveHistoryWorkflowTask(workflowName string, remain, remainDays int) error { + if remain == 0 && remainDays == 0 { + return nil + } + query := bson.M{"workflow_name": workflowName, "is_deleted": false} + count, err := c.CountDocuments(context.TODO(), query) + if err != nil { + return err + } + if remain > 0 { + query["task_id"] = bson.M{"$lt": int(count) - remain + 1} + } + if remainDays > 0 { + query["create_time"] = bson.M{"$lt": time.Now().AddDate(0, 0, -remainDays).Unix()} + } + change := bson.M{"$set": bson.M{ + "is_archived": true, + }} + _, err = c.UpdateMany(context.TODO(), query, change) + + return err +} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/workflow_v3.go b/pkg/microservice/aslan/core/common/repository/mongodb/workflow_v3.go new file mode 100644 index 0000000000000000000000000000000000000000..e0521b5947e2a6464c9da1f1730f17eef6219afc --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/mongodb/workflow_v3.go @@ -0,0 +1,163 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "errors" + "fmt" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type WorkflowV3Coll struct { + *mongo.Collection + + coll string +} + +type ListWorkflowV3Option struct { + ProjectName string +} + +func NewWorkflowV3Coll() *WorkflowV3Coll { + name := models.WorkflowV3{}.TableName() + return &WorkflowV3Coll{ + Collection: mongotool.Database(config.MongoDatabase()).Collection(name), + coll: name, + } +} + +func (c *WorkflowV3Coll) GetCollectionName() string { + return c.coll +} + +func (c *WorkflowV3Coll) EnsureIndex(ctx context.Context) error { + mod := mongo.IndexModel{ + Keys: bson.M{"project_name": 1}, + Options: options.Index().SetUnique(false), + } + + _, err := c.Indexes().CreateOne(ctx, mod) + + return err +} + +func (c *WorkflowV3Coll) Create(obj *models.WorkflowV3) (string, error) { + if obj == nil { + return "", fmt.Errorf("nil object") + } + + res, err := c.InsertOne(context.TODO(), obj) + if err != nil { + return "", err + } + ID, ok := res.InsertedID.(primitive.ObjectID) + if !ok { + return "", errors.New("failed to get object id from create") + } + return ID.Hex(), err +} + +func (c *WorkflowV3Coll) List(opt *ListWorkflowV3Option, pageNum, pageSize int64) ([]*models.WorkflowV3, int64, error) { + resp := make([]*models.WorkflowV3, 0) + query := bson.M{} + if opt.ProjectName != "" { + query["project_name"] = opt.ProjectName + } + count, err := c.CountDocuments(context.TODO(), query) + if err != nil { + return nil, 0, err + } + var findOption *options.FindOptions + if pageNum == 0 && pageSize == 0 { + findOption = options.Find() + } else { + findOption = options.Find(). + SetSkip((pageNum - 1) * pageSize). + SetLimit(pageSize) + } + + cursor, err := c.Collection.Find(context.TODO(), query, findOption) + if err != nil { + return nil, 0, err + } + err = cursor.All(context.TODO(), &resp) + if err != nil { + return nil, 0, err + } + return resp, count, nil +} + +func (c *WorkflowV3Coll) Find(name string) (*models.WorkflowV3, error) { + resp := new(models.WorkflowV3) + query := bson.M{"name": name} + + err := c.FindOne(context.TODO(), query).Decode(&resp) + if err != nil { + return nil, err + } + return resp, nil +} + +func (c *WorkflowV3Coll) GetByID(idstring string) (*models.WorkflowV3, error) { + resp := new(models.WorkflowV3) + id, err := primitive.ObjectIDFromHex(idstring) + if err != nil { + return nil, err + } + query := bson.M{"_id": id} + + err = c.FindOne(context.TODO(), query).Decode(&resp) + if err != nil { + return nil, err + } + return resp, nil +} + +func (c *WorkflowV3Coll) Update(idString string, obj *models.WorkflowV3) error { + if obj == nil { + return fmt.Errorf("nil object") + } + id, err := primitive.ObjectIDFromHex(idString) + if err != nil { + return fmt.Errorf("invalid id") + } + filter := bson.M{"_id": id} + update := bson.M{"$set": obj} + + _, err = c.UpdateOne(context.TODO(), filter, update) + return err +} + +func (c *WorkflowV3Coll) DeleteByID(idString string) error { + id, err := primitive.ObjectIDFromHex(idString) + if err != nil { + return err + } + query := bson.M{"_id": id} + + _, err = c.DeleteOne(context.TODO(), query) + return err +} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/workflow_v4.go b/pkg/microservice/aslan/core/common/repository/mongodb/workflow_v4.go new file mode 100644 index 0000000000000000000000000000000000000000..0e935a1d8e98ecfae8e0b2e05b4541baefe3a72e --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/mongodb/workflow_v4.go @@ -0,0 +1,164 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "errors" + "fmt" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type WorkflowV4Coll struct { + *mongo.Collection + + coll string +} + +type ListWorkflowV4Option struct { + ProjectName string +} + +func NewWorkflowV4Coll() *WorkflowV4Coll { + name := models.WorkflowV4{}.TableName() + return &WorkflowV4Coll{ + Collection: mongotool.Database(config.MongoDatabase()).Collection(name), + coll: name, + } +} + +func (c *WorkflowV4Coll) GetCollectionName() string { + return c.coll +} + +func (c *WorkflowV4Coll) EnsureIndex(ctx context.Context) error { + mod := mongo.IndexModel{ + Keys: bson.M{"project": 1}, + Options: options.Index().SetUnique(false), + } + + _, err := c.Indexes().CreateOne(ctx, mod) + + return err +} + +func (c *WorkflowV4Coll) Create(obj *models.WorkflowV4) (string, error) { + if obj == nil { + return "", fmt.Errorf("nil object") + } + + res, err := c.InsertOne(context.TODO(), obj) + if err != nil { + return "", err + } + ID, ok := res.InsertedID.(primitive.ObjectID) + if !ok { + return "", errors.New("failed to get object id from create") + } + return ID.Hex(), err +} + +func (c *WorkflowV4Coll) List(opt *ListWorkflowV4Option, pageNum, pageSize int64) ([]*models.WorkflowV4, int64, error) { + resp := make([]*models.WorkflowV4, 0) + query := bson.M{} + if opt.ProjectName != "" { + query["project"] = opt.ProjectName + } + + count, err := c.CountDocuments(context.TODO(), query) + if err != nil { + return nil, 0, err + } + var findOption *options.FindOptions + if pageNum == 0 && pageSize == 0 { + findOption = options.Find() + } else { + findOption = options.Find(). + SetSkip((pageNum - 1) * pageSize). + SetLimit(pageSize) + } + + cursor, err := c.Collection.Find(context.TODO(), query, findOption) + if err != nil { + return nil, 0, err + } + err = cursor.All(context.TODO(), &resp) + if err != nil { + return nil, 0, err + } + return resp, count, nil +} + +func (c *WorkflowV4Coll) Find(name string) (*models.WorkflowV4, error) { + resp := new(models.WorkflowV4) + query := bson.M{"name": name} + + err := c.FindOne(context.TODO(), query).Decode(&resp) + if err != nil { + return nil, err + } + return resp, nil +} + +func (c *WorkflowV4Coll) GetByID(idstring string) (*models.WorkflowV4, error) { + resp := new(models.WorkflowV4) + id, err := primitive.ObjectIDFromHex(idstring) + if err != nil { + return nil, err + } + query := bson.M{"_id": id} + + err = c.FindOne(context.TODO(), query).Decode(&resp) + if err != nil { + return nil, err + } + return resp, nil +} + +func (c *WorkflowV4Coll) Update(idString string, obj *models.WorkflowV4) error { + if obj == nil { + return fmt.Errorf("nil object") + } + id, err := primitive.ObjectIDFromHex(idString) + if err != nil { + return fmt.Errorf("invalid id") + } + filter := bson.M{"_id": id} + update := bson.M{"$set": obj} + + _, err = c.UpdateOne(context.TODO(), filter, update) + return err +} + +func (c *WorkflowV4Coll) DeleteByID(idString string) error { + id, err := primitive.ObjectIDFromHex(idString) + if err != nil { + return err + } + query := bson.M{"_id": id} + + _, err = c.DeleteOne(context.TODO(), query) + return err +} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/yaml_template.go b/pkg/microservice/aslan/core/common/repository/mongodb/yaml_template.go new file mode 100644 index 0000000000000000000000000000000000000000..7cfcf4917ebd53cfecd7c5d621e350572af1cfa4 --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/mongodb/yaml_template.go @@ -0,0 +1,132 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "fmt" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type YamlTemplateColl struct { + *mongo.Collection + + coll string +} + +func NewYamlTemplateColl() *YamlTemplateColl { + name := models.YamlTemplate{}.TableName() + return &YamlTemplateColl{ + Collection: mongotool.Database(config.MongoDatabase()).Collection(name), + coll: name, + } +} + +func (c *YamlTemplateColl) GetCollectionName() string { + return c.coll +} + +func (c *YamlTemplateColl) EnsureIndex(ctx context.Context) error { + mod := mongo.IndexModel{ + Keys: bson.M{"name": 1}, + Options: options.Index().SetUnique(true), + } + + _, err := c.Indexes().CreateOne(ctx, mod) + + return err +} + +func (c *YamlTemplateColl) Create(obj *models.YamlTemplate) error { + if obj == nil { + return fmt.Errorf("nil object") + } + + _, err := c.InsertOne(context.TODO(), obj) + return err +} + +func (c *YamlTemplateColl) Update(idString string, obj *models.YamlTemplate) error { + if obj == nil { + return fmt.Errorf("nil object") + } + id, err := primitive.ObjectIDFromHex(idString) + if err != nil { + return fmt.Errorf("invalid id") + } + filter := bson.M{"_id": id} + update := bson.M{"$set": obj} + + _, err = c.UpdateOne(context.TODO(), filter, update) + return err +} + +func (c *YamlTemplateColl) List(pageNum, pageSize int) ([]*models.YamlTemplate, int, error) { + resp := make([]*models.YamlTemplate, 0) + query := bson.M{} + count, err := c.CountDocuments(context.TODO(), query) + if err != nil { + return nil, 0, err + } + opt := options.Find(). + SetSkip(int64((pageNum - 1) * pageSize)). + SetLimit(int64(pageSize)) + + cursor, err := c.Collection.Find(context.TODO(), query, opt) + if err != nil { + return nil, 0, err + } + err = cursor.All(context.TODO(), &resp) + if err != nil { + return nil, 0, err + } + return resp, int(count), nil +} + +func (c *YamlTemplateColl) GetById(idstring string) (*models.YamlTemplate, error) { + resp := new(models.YamlTemplate) + id, err := primitive.ObjectIDFromHex(idstring) + if err != nil { + return nil, err + } + query := bson.M{"_id": id} + + err = c.FindOne(context.TODO(), query).Decode(&resp) + if err != nil { + return nil, err + } + return resp, nil +} + +func (c *YamlTemplateColl) DeleteByID(idstring string) error { + id, err := primitive.ObjectIDFromHex(idstring) + if err != nil { + return err + } + query := bson.M{"_id": id} + + _, err = c.DeleteOne(context.TODO(), query) + return err +} diff --git a/pkg/microservice/aslan/core/common/service/base/convert.go b/pkg/microservice/aslan/core/common/service/base/convert.go index b691354808ebed368ce70555c71ae619c29d397c..db6460a1692138606ead8fa16d3619c853672262 100644 --- a/pkg/microservice/aslan/core/common/service/base/convert.go +++ b/pkg/microservice/aslan/core/common/service/base/convert.go @@ -31,7 +31,15 @@ type Preview struct { func ToPreview(sb map[string]interface{}) (*Preview, error) { var pre *Preview if err := task.IToi(sb, &pre); err != nil { - return nil, fmt.Errorf("convert interface to SubTaskPreview error: %v", err) + return nil, fmt.Errorf("convert interface to SubTaskPreview error: %s", err) + } + return pre, nil +} + +func ToScanning(sb map[string]interface{}) (*task.Scanning, error) { + var pre *task.Scanning + if err := task.IToi(sb, &pre); err != nil { + return nil, fmt.Errorf("convert interface to Scanning error: %s", err) } return pre, nil } @@ -39,7 +47,7 @@ func ToPreview(sb map[string]interface{}) (*Preview, error) { func ToBuildTask(sb map[string]interface{}) (*task.Build, error) { var t *task.Build if err := task.IToi(sb, &t); err != nil { - return nil, fmt.Errorf("convert interface to BuildTaskV2 error: %v", err) + return nil, fmt.Errorf("convert interface to BuildTaskV2 error: %s", err) } return t, nil } @@ -92,6 +100,14 @@ func ToReleaseImageTask(sb map[string]interface{}) (*task.ReleaseImage, error) { return t, nil } +func ToArtifactPackageImageTask(sb map[string]interface{}) (*task.ArtifactPackage, error) { + var t *task.ArtifactPackage + if err := task.IToi(sb, &t); err != nil { + return nil, fmt.Errorf("convert interface to ReleaseImageTask error: %v", err) + } + return t, nil +} + func ToJiraTask(sb map[string]interface{}) (*task.Jira, error) { var t *task.Jira if err := task.IToi(sb, &t); err != nil { @@ -115,3 +131,19 @@ func ToJenkinsBuildTask(sb map[string]interface{}) (*task.JenkinsBuild, error) { } return jenkinsBuild, nil } + +func ToTriggerTask(sb map[string]interface{}) (*task.Trigger, error) { + var trigger *task.Trigger + if err := task.IToi(sb, &trigger); err != nil { + return nil, fmt.Errorf("convert interface to triggerTask error: %s", err) + } + return trigger, nil +} + +func ToExtensionTask(sb map[string]interface{}) (*task.Extension, error) { + var extension *task.Extension + if err := task.IToi(sb, &extension); err != nil { + return nil, fmt.Errorf("convert interface to extensionTask error: %s", err) + } + return extension, nil +} diff --git a/pkg/microservice/aslan/core/common/service/build.go b/pkg/microservice/aslan/core/common/service/build.go index db16a3dd4948ed99df5d827be96cc20ba706f5dc..ea453e6517a4eafab7c1d75eab2fa23a5c519dce 100644 --- a/pkg/microservice/aslan/core/common/service/build.go +++ b/pkg/microservice/aslan/core/common/service/build.go @@ -24,8 +24,10 @@ import ( commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + commonutil "github.com/koderover/zadig/pkg/microservice/aslan/core/common/util" "github.com/koderover/zadig/pkg/setting" e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/types" ) @@ -33,6 +35,9 @@ func CreateBuild(username string, build *commonmodels.Build, log *zap.SugaredLog if len(build.Name) == 0 { return e.ErrCreateBuildModule.AddDesc("empty name") } + if err := commonutil.CheckDefineResourceParam(build.PreBuild.ResReq, build.PreBuild.ResReqSpec); err != nil { + return e.ErrCreateBuildModule.AddDesc(err.Error()) + } build.UpdateBy = username correctFields(build) @@ -49,6 +54,9 @@ func UpdateBuild(username string, build *commonmodels.Build, log *zap.SugaredLog if len(build.Name) == 0 { return e.ErrUpdateBuildModule.AddDesc("empty name") } + if err := commonutil.CheckDefineResourceParam(build.PreBuild.ResReq, build.PreBuild.ResReqSpec); err != nil { + return e.ErrUpdateBuildModule.AddDesc(err.Error()) + } existed, err := commonrepo.NewBuildColl().Find(&commonrepo.BuildFindOption{Name: build.Name, ProductName: build.ProductName}) if err == nil && existed.PreBuild != nil && build.PreBuild != nil { @@ -109,9 +117,10 @@ func EnsureResp(build *commonmodels.Build) { if len(build.Targets) == 0 { build.Targets = make([]*commonmodels.ServiceModuleTarget, 0) } + build.Repos = build.SafeRepos() - if len(build.Repos) == 0 { - build.Repos = make([]*types.Repository, 0) + for _, repo := range build.Repos { + repo.RepoNamespace = repo.GetRepoNamespace() } if build.PreBuild != nil { @@ -134,4 +143,67 @@ func EnsureResp(build *commonmodels.Build) { build.PreBuild.Parameters = make([]*commonmodels.Parameter, 0) } } + + if build.TemplateID != "" { + buildTemplate, err := commonrepo.NewBuildTemplateColl().Find(&commonrepo.BuildTemplateQueryOption{ + ID: build.TemplateID, + }) + //NOTE deleted template should not block the normal logic of build modules + if err != nil { + log.Warnf("failed to find build template with id: %s, err: %s", build.TemplateID, err) + } + build.TargetRepos = make([]*commonmodels.TargetRepo, 0, len(build.Targets)) + for _, target := range build.Targets { + for _, repo := range target.Repos { + repo.RepoNamespace = repo.GetRepoNamespace() + } + envs := target.Envs + if buildTemplate != nil { + envs = MergeBuildEnvs(buildTemplate.PreBuild.Envs, envs) + } + targetRepo := &commonmodels.TargetRepo{ + Service: &commonmodels.ServiceModuleTargetBase{ + ProductName: target.ProductName, + ServiceName: target.ServiceName, + ServiceModule: target.ServiceModule, + }, + Repos: target.Repos, + Envs: envs, + } + for _, v := range targetRepo.Envs { + if v.IsCredential { + v.Value = setting.MaskValue + } + } + build.TargetRepos = append(build.TargetRepos, targetRepo) + } + } +} + +func FindReposByTarget(projectName, serviceName, serviceModule string, build *commonmodels.Build) []*types.Repository { + if build.TemplateID == "" { + return build.SafeRepos() + } + for _, target := range build.Targets { + if target.ServiceName == serviceName && target.ProductName == projectName && target.ServiceModule == serviceModule { + return target.Repos + } + } + return build.SafeRepos() +} + +func MergeBuildEnvs(templateEnvs []*commonmodels.KeyVal, customEnvs []*commonmodels.KeyVal) []*commonmodels.KeyVal { + customEnvMap := make(map[string]*commonmodels.KeyVal) + for _, v := range customEnvs { + customEnvMap[v.Key] = v + } + retEnvs := make([]*commonmodels.KeyVal, 0) + for _, v := range templateEnvs { + if cv, ok := customEnvMap[v.Key]; ok { + retEnvs = append(retEnvs, cv) + } else { + retEnvs = append(retEnvs, v) + } + } + return retEnvs } diff --git a/pkg/microservice/aslan/core/common/service/callback.go b/pkg/microservice/aslan/core/common/service/callback.go new file mode 100644 index 0000000000000000000000000000000000000000..cd9da86654a234748b5300e33da7f408bfd9738b --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/callback.go @@ -0,0 +1,26 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" +) + +func HandleCallback(req *commonmodels.CallbackRequest) error { + return commonrepo.NewCallbackRequestColl().Create(req) +} diff --git a/pkg/microservice/aslan/core/common/service/codehub/client.go b/pkg/microservice/aslan/core/common/service/codehub/client.go index 8503b01cdb9450f2dad120ff87ce46916d7a1176..74ca22882fd55bc6e8263990f309947a52739c03 100644 --- a/pkg/microservice/aslan/core/common/service/codehub/client.go +++ b/pkg/microservice/aslan/core/common/service/codehub/client.go @@ -24,8 +24,8 @@ type Client struct { *codehub.CodeHubClient } -func NewClient(ak, sk, region string) *Client { - c := codehub.NewCodeHubClient(ak, sk, region) +func NewClient(ak, sk, region, proxyAddr string, enableProxy bool) *Client { + c := codehub.NewCodeHubClient(ak, sk, region, proxyAddr, enableProxy) return &Client{ CodeHubClient: c, } diff --git a/pkg/microservice/aslan/core/common/service/collaboration/collaboration_mode.go b/pkg/microservice/aslan/core/common/service/collaboration/collaboration_mode.go new file mode 100644 index 0000000000000000000000000000000000000000..1d127a60851f7c2b2e5c91d6d8cc5b6b51e2e9d7 --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/collaboration/collaboration_mode.go @@ -0,0 +1,106 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package collaboration + +import ( + "go.mongodb.org/mongo-driver/mongo" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/collaboration/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/collaboration/repository/mongodb" +) + +type GetCollaborationModeResp struct { + Collaborations []*models.CollaborationMode `json:"collaborations"` +} + +func BuildEnvCMMapKey(projectName string, envName string) string { + return projectName + " " + envName +} + +func BuildWorkflowCMMapKey(projectName string, workflowName string) string { + return projectName + " " + workflowName +} + +func GetEnvCMMap(projects []string, logger *zap.SugaredLogger) (map[string]sets.String, error) { + collaborationModes, err := GetCollaborationModes(projects, logger) + if err != nil { + logger.Errorf("GetCollaborationModes error: %v", err) + return nil, err + } + return buildEnvCMMap(collaborationModes.Collaborations), nil +} + +func GetWorkflowCMMap(projects []string, logger *zap.SugaredLogger) (map[string]sets.String, error) { + collaborationModes, err := GetCollaborationModes(projects, logger) + if err != nil { + logger.Errorf("GetCollaborationModes error: %v", err) + return nil, err + } + return buildWorkflowCMMap(collaborationModes.Collaborations), nil +} + +func buildEnvCMMap(collaborations []*models.CollaborationMode) map[string]sets.String { + envCMMap := make(map[string]sets.String) + for _, cm := range collaborations { + for _, product := range cm.Products { + key := BuildEnvCMMapKey(cm.ProjectName, product.Name) + if cmSet, ok := envCMMap[key]; ok { + cmSet.Insert(cm.Name) + envCMMap[key] = cmSet + } else { + cmSet := sets.NewString(cm.Name) + envCMMap[key] = cmSet + } + } + } + return envCMMap +} + +func buildWorkflowCMMap(collaborations []*models.CollaborationMode) map[string]sets.String { + workflowCMMap := make(map[string]sets.String) + for _, cm := range collaborations { + for _, workflow := range cm.Workflows { + key := BuildWorkflowCMMapKey(cm.ProjectName, workflow.Name) + if cmSet, ok := workflowCMMap[key]; ok { + cmSet.Insert(cm.Name) + workflowCMMap[key] = cmSet + } else { + cmSet := sets.NewString(cm.Name) + workflowCMMap[key] = cmSet + } + } + } + return workflowCMMap +} + +func GetCollaborationModes(projects []string, logger *zap.SugaredLogger) (*GetCollaborationModeResp, error) { + collaborations, err := mongodb.NewCollaborationModeColl().List(&mongodb.CollaborationModeListOptions{ + Projects: projects, + }) + if err != nil { + if err == mongo.ErrNoDocuments { + return &GetCollaborationModeResp{}, nil + } + logger.Errorf("GetCollaborationModes error, err msg:%s", err) + return nil, err + } + return &GetCollaborationModeResp{ + Collaborations: collaborations, + }, nil +} diff --git a/pkg/microservice/aslan/core/common/service/collie/client.go b/pkg/microservice/aslan/core/common/service/collie/client.go index a2b94c6a0ad6fc85cd226f9b9fd80b7fbb21d830..05c9beaff788e84546206096bafa2134ee111377 100644 --- a/pkg/microservice/aslan/core/common/service/collie/client.go +++ b/pkg/microservice/aslan/core/common/service/collie/client.go @@ -17,27 +17,22 @@ limitations under the License. package collie import ( - "github.com/koderover/zadig/pkg/setting" "github.com/koderover/zadig/pkg/tool/httpclient" ) type Client struct { *httpclient.Client - host string - token string + host string } -func New(host, token string) *Client { +func New(host string) *Client { c := httpclient.New( - httpclient.SetAuthScheme(setting.RootAPIKey), - httpclient.SetAuthToken(token), httpclient.SetHostURL(host), ) return &Client{ Client: c, host: host, - token: token, } } diff --git a/pkg/microservice/aslan/core/common/service/command/git_cmd.go b/pkg/microservice/aslan/core/common/service/command/git_cmd.go index d47f3f8cbbc32066e478d2e5f9a647b266508321..0fb75f0f798997d1cd6b1cd1f9e4a4a5a98d00fb 100644 --- a/pkg/microservice/aslan/core/common/service/command/git_cmd.go +++ b/pkg/microservice/aslan/core/common/service/command/git_cmd.go @@ -30,7 +30,7 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/codehost" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" "github.com/koderover/zadig/pkg/tool/log" ) @@ -38,6 +38,7 @@ type Repo struct { Source string `yaml:"source"` Address string `yaml:"address"` Owner string `yaml:"owner"` + Namespace string `yaml:"namespace"` Name string `yaml:"name"` RemoteName string `yaml:"remote_name"` Branch string `yaml:"branch"` @@ -65,7 +66,7 @@ type Command struct { IgnoreError bool } -func RunGitCmds(codehostDetail *codehost.Detail, repoOwner, repoName, branchName, remoteName string) error { +func RunGitCmds(codehostDetail *systemconfig.CodeHost, repoOwner, repoNamespace, repoName, branchName, remoteName string) error { var ( tokens []string repo *Repo @@ -73,11 +74,12 @@ func RunGitCmds(codehostDetail *codehost.Detail, repoOwner, repoName, branchName cmds = make([]*Command, 0) ) repo = &Repo{ - Source: codehostDetail.Source, + Source: codehostDetail.Type, Address: codehostDetail.Address, Name: repoName, + Namespace: repoNamespace, Branch: branchName, - OauthToken: codehostDetail.OauthToken, + OauthToken: codehostDetail.AccessToken, RemoteName: remoteName, Owner: repoOwner, } @@ -97,7 +99,7 @@ func RunGitCmds(codehostDetail *codehost.Detail, repoOwner, repoName, branchName tokens = append(tokens, repo.OauthToken) cmds = append(cmds, buildGitCommands(repo)...) - if repo.Source == setting.SourceFromGithub { + if codehostDetail.EnableProxy { httpsProxy := config.ProxyHTTPSAddr() httpProxy := config.ProxyHTTPAddr() if httpsProxy != "" { @@ -163,6 +165,10 @@ func buildGitCommands(repo *Repo) []*Command { os.MkdirAll(workDir, 0777) } + if strings.Contains(repoName, "-new") { + repo.Name = strings.TrimSuffix(repo.Name, "-new") + } + // 预防非正常退出导致git被锁住 _ = os.Remove(path.Join(workDir, "/.git/index.lock")) @@ -172,7 +178,7 @@ func buildGitCommands(repo *Repo) []*Command { cmds = append(cmds, &Command{Cmd: RemoteRemove(repo.RemoteName), DisableTrace: true, IgnoreError: true}) } - if repo.Source == setting.SourceFromGitlab { + if repo.Source == setting.SourceFromGitlab || repo.Source == setting.SourceFromGitee { u, _ := url.Parse(repo.Address) url := OAuthCloneURL(repo.OauthToken, u.Host, repo.Owner, repo.Name, u.Scheme) cmds = append(cmds, &Command{ diff --git a/pkg/microservice/aslan/core/common/service/config_payload.go b/pkg/microservice/aslan/core/common/service/config_payload.go index 7c15bd52d09cd98f99cb25c03ad03d1262a10bf0..bf94e036ac728ca16561a2a6acd723872de2e33a 100644 --- a/pkg/microservice/aslan/core/common/service/config_payload.go +++ b/pkg/microservice/aslan/core/common/service/config_payload.go @@ -21,7 +21,8 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/codehost" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" + "github.com/koderover/zadig/pkg/tool/crypto" ) func GetConfigPayload(codeHostID int) *models.ConfigPayload { @@ -54,6 +55,7 @@ func GetConfigPayload(codeHostID int) *models.ConfigPayload { ReaperImage: config.ReaperImage(), ReaperBinaryFile: config.ReaperBinaryFile(), PredatorImage: config.PredatorImage(), + PackagerImage: config.PackagerImage(), }, Docker: models.DockerConfig{ HostList: config.DockerHosts(), @@ -64,6 +66,8 @@ func GetConfigPayload(codeHostID int) *models.ConfigPayload { JenkinsBuildConfig: models.JenkinsBuildConfig{ JenkinsBuildImage: config.JenkinsImage(), }, + AesKey: crypto.GetAesKey(), + BuildConcurrency: 5, } githubApps, _ := mongodb.NewGithubAppColl().Find() @@ -73,7 +77,7 @@ func GetConfigPayload(codeHostID int) *models.ConfigPayload { } if codeHostID > 0 { - ch, _ := codehost.GetCodeHostInfoByID(codeHostID) + ch, _ := systemconfig.New().GetCodeHost(codeHostID) if ch != nil && ch.Type == setting.SourceFromGithub { payload.Github.AccessToken = ch.AccessToken } @@ -84,10 +88,40 @@ func GetConfigPayload(codeHostID int) *models.ConfigPayload { payload.Proxy = *proxies[0] } + concurrencySettings, _ := mongodb.NewSystemSettingColl().Get() + if concurrencySettings != nil { + payload.BuildConcurrency = concurrencySettings.BuildConcurrency + } + privateKeys, _ := mongodb.NewPrivateKeyColl().List(&mongodb.PrivateKeyArgs{}) if len(privateKeys) != 0 { payload.PrivateKeys = privateKeys } + k8sClusters, _ := mongodb.NewK8SClusterColl().List(nil) + if len(k8sClusters) != 0 { + var K8SClusterResp []*models.K8SClusterResp + for _, k8sCluster := range k8sClusters { + K8SClusterResp = append(K8SClusterResp, &models.K8SClusterResp{ + ID: k8sCluster.ID.Hex(), + Name: k8sCluster.Name, + AdvancedConfig: k8sCluster.AdvancedConfig, + }) + } + payload.K8SClusters = K8SClusterResp + } + + objectStorage, _ := mongodb.NewS3StorageColl().FindDefault() + if objectStorage != nil { + payload.S3Storage.Endpoint = objectStorage.Endpoint + payload.S3Storage.Sk = objectStorage.Sk + payload.S3Storage.Ak = objectStorage.Ak + payload.S3Storage.Path = objectStorage.Subfolder + payload.S3Storage.Bucket = objectStorage.Bucket + payload.S3Storage.Protocol = "https" + if objectStorage.Insecure { + payload.S3Storage.Protocol = "http" + } + } return payload } diff --git a/pkg/microservice/aslan/core/common/service/delivery.go b/pkg/microservice/aslan/core/common/service/delivery.go index 91d40737f6f19b0e59b991f6750ae8c74f47ba08..1e2c02bc6be2fef1479f5bf78fe7e5b016add65e 100644 --- a/pkg/microservice/aslan/core/common/service/delivery.go +++ b/pkg/microservice/aslan/core/common/service/delivery.go @@ -26,7 +26,6 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" taskmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" @@ -36,30 +35,30 @@ import ( ) func DeleteDeliveryInfos(productName string, log *zap.SugaredLogger) error { - deliveryVersions, err := mongodb.NewDeliveryVersionColl().ListDeliveryVersions(productName, 1) + deliveryVersions, err := commonrepo.NewDeliveryVersionColl().ListDeliveryVersions(productName) if err != nil { log.Errorf("delete DeleteDeliveryInfo error: %v", err) return e.ErrDeleteDeliveryVersion } errList := new(multierror.Error) for _, deliveryVersion := range deliveryVersions { - err = mongodb.NewDeliveryVersionColl().Delete(deliveryVersion.ID.Hex()) + err = commonrepo.NewDeliveryVersionColl().Delete(deliveryVersion.ID.Hex()) if err != nil { errList = multierror.Append(errList, fmt.Errorf("DeliveryVersion delete %s error: %v", deliveryVersion.ID.String(), err)) } - err = mongodb.NewDeliveryBuildColl().Delete(deliveryVersion.ID.Hex()) + err = commonrepo.NewDeliveryBuildColl().Delete(deliveryVersion.ID.Hex()) if err != nil { errList = multierror.Append(errList, fmt.Errorf("DeliveryBuild delete %s error: %v", deliveryVersion.ID.String(), err)) } - err = mongodb.NewDeliveryDeployColl().Delete(deliveryVersion.ID.Hex()) + err = commonrepo.NewDeliveryDeployColl().Delete(deliveryVersion.ID.Hex()) if err != nil { errList = multierror.Append(errList, fmt.Errorf("DeliveryDeploy delete %s error: %v", deliveryVersion.ID.String(), err)) } - err = mongodb.NewDeliveryTestColl().Delete(deliveryVersion.ID.Hex()) + err = commonrepo.NewDeliveryTestColl().Delete(deliveryVersion.ID.Hex()) if err != nil { errList = multierror.Append(errList, fmt.Errorf("DeliveryTest delete %s error: %v", deliveryVersion.ID.String(), err)) } - err = mongodb.NewDeliveryDistributeColl().Delete(deliveryVersion.ID.Hex()) + err = commonrepo.NewDeliveryDistributeColl().Delete(deliveryVersion.ID.Hex()) if err != nil { errList = multierror.Append(errList, fmt.Errorf("DeliveryDistribute delete %s error: %v", deliveryVersion.ID.String(), err)) } @@ -71,9 +70,8 @@ func DeleteDeliveryInfos(productName string, log *zap.SugaredLogger) error { return nil } -func AddDeliveryVersion(orgID, taskID int, productName, workflowName string, pipelineTask *taskmodels.Task, logger *zap.SugaredLogger) error { - deliveryVersionArgs := &mongodb.DeliveryVersionArgs{ - OrgID: orgID, +func AddDeliveryVersion(taskID int, productName, workflowName string, pipelineTask *taskmodels.Task, logger *zap.SugaredLogger) error { + deliveryVersionArgs := &commonrepo.DeliveryVersionArgs{ ProductName: productName, WorkflowName: workflowName, TaskID: taskID, @@ -87,7 +85,6 @@ func AddDeliveryVersion(orgID, taskID int, productName, workflowName string, pip } deliveryVersion := new(commonmodels.DeliveryVersion) - deliveryVersion.OrgID = orgID deliveryVersion.WorkflowName = workflowName deliveryVersion.TaskID = taskID deliveryVersion.ProductName = productName @@ -110,8 +107,8 @@ func AddDeliveryVersion(orgID, taskID int, productName, workflowName string, pip return err } -func GetDeliveryVersion(args *mongodb.DeliveryVersionArgs, log *zap.SugaredLogger) (*commonmodels.DeliveryVersion, error) { - resp, err := mongodb.NewDeliveryVersionColl().Get(args) +func GetDeliveryVersion(args *commonrepo.DeliveryVersionArgs, log *zap.SugaredLogger) (*commonmodels.DeliveryVersion, error) { + resp, err := commonrepo.NewDeliveryVersionColl().Get(args) if err != nil { log.Errorf("get deliveryVersion error: %v", err) return nil, e.ErrGetDeliveryVersion @@ -120,7 +117,7 @@ func GetDeliveryVersion(args *mongodb.DeliveryVersionArgs, log *zap.SugaredLogge } func InsertDeliveryVersion(args *commonmodels.DeliveryVersion, log *zap.SugaredLogger) error { - err := mongodb.NewDeliveryVersionColl().Insert(args) + err := commonrepo.NewDeliveryVersionColl().Insert(args) if err != nil { log.Errorf("insert deliveryVersion error: %v", err) return e.ErrCreateDeliveryVersion @@ -377,10 +374,12 @@ func getProductEnvInfo(pipelineTask *taskmodels.Task, log *zap.SugaredLogger) (* product.Namespace = productInfo.Namespace } - if renderSet, err := GetRenderSet(product.Namespace, pipelineTask.Render.Revision, log); err == nil { - product.Vars = renderSet.KVs - } else { - log.Errorf("GetProductEnvInfo GetRenderSet namespace:%s pipelineTask.Render.Revision:%d err:%v", product.GetNamespace(), pipelineTask.Render.Revision, err) + if pipelineTask.Render != nil { + if renderSet, err := GetRenderSet(product.Namespace, pipelineTask.Render.Revision, false, product.EnvName, log); err == nil { + product.Vars = renderSet.KVs + } else { + log.Warnf("GetProductEnvInfo GetRenderSet namespace:%s pipelineTask.Render.Revision:%d err:%v", product.GetNamespace(), pipelineTask.Render.Revision, err) + } } //返回中的ProductName即产品模板的名称 @@ -391,7 +390,7 @@ func getProductEnvInfo(pipelineTask *taskmodels.Task, log *zap.SugaredLogger) (* } func insertDeliveryBuild(args *commonmodels.DeliveryBuild, log *zap.SugaredLogger) error { - err := mongodb.NewDeliveryBuildColl().Insert(args) + err := commonrepo.NewDeliveryBuildColl().Insert(args) if err != nil { log.Errorf("insert deliveryBuild error: %v", err) return e.ErrCreateDeliveryBuild @@ -400,7 +399,7 @@ func insertDeliveryBuild(args *commonmodels.DeliveryBuild, log *zap.SugaredLogge } func insertDeliveryDeploy(args *commonmodels.DeliveryDeploy, log *zap.SugaredLogger) error { - err := mongodb.NewDeliveryDeployColl().Insert(args) + err := commonrepo.NewDeliveryDeployColl().Insert(args) if err != nil { log.Errorf("insert deliveryDeploy error: %v", err) return e.ErrCreateDeliveryDeploy @@ -409,7 +408,7 @@ func insertDeliveryDeploy(args *commonmodels.DeliveryDeploy, log *zap.SugaredLog } func InsertDeliveryTest(args *commonmodels.DeliveryTest, log *zap.SugaredLogger) error { - err := mongodb.NewDeliveryTestColl().Insert(args) + err := commonrepo.NewDeliveryTestColl().Insert(args) if err != nil { log.Errorf("insert deliveryTest error: %v", err) return e.ErrCreateDeliveryTest @@ -418,7 +417,7 @@ func InsertDeliveryTest(args *commonmodels.DeliveryTest, log *zap.SugaredLogger) } func insertDeliveryDistribute(args *commonmodels.DeliveryDistribute, log *zap.SugaredLogger) error { - err := mongodb.NewDeliveryDistributeColl().Insert(args) + err := commonrepo.NewDeliveryDistributeColl().Insert(args) if err != nil { log.Errorf("insert deliveryDistribute error: %v", err) return e.ErrCreateDeliveryDistribute @@ -427,7 +426,7 @@ func insertDeliveryDistribute(args *commonmodels.DeliveryDistribute, log *zap.Su } func updateDeliveryVersion(args *commonmodels.DeliveryVersion, log *zap.SugaredLogger) error { - err := mongodb.NewDeliveryVersionColl().Update(args) + err := commonrepo.NewDeliveryVersionColl().Update(args) if err != nil { log.Errorf("update deliveryVersion error: %v", err) return e.ErrUpdateDeliveryVersion @@ -453,8 +452,13 @@ func updateServiceImage(serviceName, image, containerName string, product *commo func getServiceRenderYAML(productInfo *commonmodels.Product, containers []*commonmodels.Container, serviceName, deployType string, log *zap.SugaredLogger) (string, error) { if deployType == setting.K8SDeployType { - opt := &mongodb.RenderSetFindOption{Name: productInfo.Render.Name, Revision: productInfo.Render.Revision} - newRender, err := mongodb.NewRenderSetColl().Find(opt) + opt := &commonrepo.RenderSetFindOption{ + Name: productInfo.Render.Name, + Revision: productInfo.Render.Revision, + EnvName: productInfo.EnvName, + ProductTmpl: productInfo.ProductName, + } + newRender, err := commonrepo.NewRenderSetColl().Find(opt) if err != nil { log.Errorf("[%s][P:%s]renderset Find error: %v", productInfo.EnvName, productInfo.ProductName, err) return "", fmt.Errorf("get pure yaml %s error: %v", serviceName, err) @@ -465,13 +469,13 @@ func getServiceRenderYAML(productInfo *commonmodels.Product, containers []*commo return "", fmt.Errorf("service %s not found", serviceName) } // 获取服务模板 - serviceFindOption := &mongodb.ServiceFindOption{ + serviceFindOption := &commonrepo.ServiceFindOption{ ServiceName: serviceName, ProductName: serviceInfo.ProductName, Type: setting.K8SDeployType, Revision: serviceInfo.Revision, } - svcTmpl, err := mongodb.NewServiceColl().Find(serviceFindOption) + svcTmpl, err := commonrepo.NewServiceColl().Find(serviceFindOption) if err != nil { return "", fmt.Errorf("service template %s error: %v", serviceName, err) } diff --git a/pkg/microservice/aslan/core/common/service/environment.go b/pkg/microservice/aslan/core/common/service/environment.go index 3844e98b17793b0e6a4f14fe0061048d1cef3a03..a827d31c60f96672fadc9cd5482b7f7c851d5c03 100644 --- a/pkg/microservice/aslan/core/common/service/environment.go +++ b/pkg/microservice/aslan/core/common/service/environment.go @@ -24,25 +24,32 @@ import ( "go.uber.org/zap" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/koderover/zadig/pkg/microservice/aslan/config" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" templatemodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" "github.com/koderover/zadig/pkg/shared/kube/resource" "github.com/koderover/zadig/pkg/shared/kube/wrapper" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/kube/getter" + "github.com/koderover/zadig/pkg/tool/kube/informer" "github.com/koderover/zadig/pkg/util" jsonutil "github.com/koderover/zadig/pkg/util/json" ) // FillProductTemplateValuesYamls 返回renderSet中的renderChart信息 func FillProductTemplateValuesYamls(tmpl *templatemodels.Product, log *zap.SugaredLogger) error { - renderSet, err := GetRenderSet(tmpl.ProductName, 0, log) + renderSet, err := GetRenderSet(tmpl.ProductName, 0, true, "", log) if err != nil { log.Errorf("Failed to find render set for product template %s", tmpl.ProductName) return err @@ -59,9 +66,11 @@ func FillProductTemplateValuesYamls(tmpl *templatemodels.Product, log *zap.Sugar continue } tmpl.ChartInfos = append(tmpl.ChartInfos, &templatemodels.RenderChart{ - ServiceName: renderChart.ServiceName, - ChartVersion: renderChart.ChartVersion, - ValuesYaml: renderChart.ValuesYaml, + ServiceName: renderChart.ServiceName, + ChartVersion: renderChart.ChartVersion, + ValuesYaml: renderChart.ValuesYaml, + OverrideYaml: renderChart.OverrideYaml, + OverrideValues: renderChart.OverrideValues, }) } @@ -70,38 +79,115 @@ func FillProductTemplateValuesYamls(tmpl *templatemodels.Product, log *zap.Sugar // 产品列表页服务Response type ServiceResp struct { - ServiceName string `json:"service_name"` - ServiceDisplayName string `json:"service_display_name"` - Type string `json:"type"` - Status string `json:"status"` - Images []string `json:"images,omitempty"` - ProductName string `json:"product_name"` - EnvName string `json:"env_name"` - Ingress *IngressInfo `json:"ingress"` - Ready string `json:"ready"` - EnvStatuses []*models.EnvStatus `json:"env_statuses,omitempty"` - WorkLoadType string `json:"workLoadType"` + ServiceName string `json:"service_name"` + ServiceDisplayName string `json:"service_display_name"` + Type string `json:"type"` + Status string `json:"status"` + Images []string `json:"images,omitempty"` + ProductName string `json:"product_name"` + EnvName string `json:"env_name"` + Ingress *IngressInfo `json:"ingress"` + //deprecated + Ready string `json:"ready"` + EnvStatuses []*models.EnvStatus `json:"env_statuses,omitempty"` + WorkLoadType string `json:"workLoadType"` + Revision int64 `json:"revision"` + EnvConfigs []*models.EnvConfig `json:"env_configs"` } type IngressInfo struct { HostInfo []resource.HostInfo `json:"host_info"` } -// fill service display name if necessary -func fillServiceDisplayName(svcList []*ServiceResp, productInfo *models.Product) { - if productInfo.Source == setting.SourceFromHelm { - for _, svc := range svcList { - svc.ServiceDisplayName = util.ExtraServiceName(svc.ServiceName, productInfo.Namespace) +func UnMarshalSourceDetail(source interface{}) (*models.CreateFromRepo, error) { + bs, err := json.Marshal(source) + if err != nil { + return nil, err + } + ret := &models.CreateFromRepo{} + err = json.Unmarshal(bs, ret) + return ret, err +} + +func FillGitNamespace(yamlData *templatemodels.CustomYaml) error { + if yamlData == nil || yamlData.Source != setting.SourceFromGitRepo { + return nil + } + sourceDetail, err := UnMarshalSourceDetail(yamlData.SourceDetail) + if err != nil { + return err + } + if sourceDetail.GitRepoConfig == nil { + return nil + } + sourceDetail.GitRepoConfig.Namespace = sourceDetail.GitRepoConfig.GetNamespace() + yamlData.SourceDetail = sourceDetail + return nil +} + +func GetRenderCharts(productName, envName, serviceName string, log *zap.SugaredLogger) ([]*RenderChartArg, error) { + + renderSetName := GetProductEnvNamespace(envName, productName, "") + + opt := &commonrepo.RenderSetFindOption{ + ProductTmpl: productName, + EnvName: envName, + Name: renderSetName, + } + rendersetObj, existed, err := commonrepo.NewRenderSetColl().FindRenderSet(opt) + if err != nil { + return nil, err + } + + if !existed { + return nil, nil + } + + ret := make([]*RenderChartArg, 0) + + matchedRenderChartModels := make([]*templatemodels.RenderChart, 0) + if len(serviceName) == 0 { + matchedRenderChartModels = rendersetObj.ChartInfos + } else { + serverList := strings.Split(serviceName, ",") + stringSet := sets.NewString(serverList...) + for _, singleChart := range rendersetObj.ChartInfos { + if !stringSet.Has(singleChart.ServiceName) { + continue + } + matchedRenderChartModels = append(matchedRenderChartModels, singleChart) } } + + for _, singleChart := range matchedRenderChartModels { + rcaObj := new(RenderChartArg) + rcaObj.LoadFromRenderChartModel(singleChart) + rcaObj.EnvName = envName + err = FillGitNamespace(rendersetObj.YamlData) + if err != nil { + // Note, since user can always reselect the git info, error should not block normal logic + log.Warnf("failed to fill git namespace data, err: %s", err) + } + rcaObj.YamlData = singleChart.OverrideYaml + ret = append(ret, rcaObj) + } + return ret, nil +} + +// fill service display name if necessary +func fillServiceInfo(svcList []*ServiceResp, productInfo *models.Product) { + if productInfo.Source != setting.SourceFromHelm { + return + } + for _, svc := range svcList { + svc.ServiceDisplayName = svc.ServiceName + } } -// ListWorkloadsInEnv returns all workloads in the given env which meat the filter. +// ListWorkloadsInEnv returns all workloads in the given env which meet the filter. // A filter is in this format: a=b,c=d, and it is a fuzzy matching. Which means it will return all records with a field called // a and the value contain character b. func ListWorkloadsInEnv(envName, productName, filter string, perPage, page int, log *zap.SugaredLogger) (int, []*ServiceResp, error) { - log.Infof("Start to list workloads for env %s for project %s", envName, productName) - defer log.Info("Finish to list workloads") opt := &commonrepo.ProductFindOptions{Name: productName, EnvName: envName} productInfo, err := commonrepo.NewProductColl().Find(opt) @@ -130,29 +216,61 @@ func ListWorkloadsInEnv(envName, productName, filter string, perPage, page int, } productServiceNames := sets.NewString() for _, productService := range productServices { - productServiceNames.Insert(fmt.Sprintf("%s/%s", productService.WorkloadType, productService.ServiceName)) + productServiceNames.Insert(productService.ServiceName) + } + // add services in external env data + servicesInExternalEnv, _ := commonrepo.NewServicesInExternalEnvColl().List(&commonrepo.ServicesInExternalEnvArgs{ + ProductName: productName, + EnvName: envName, + }) + for _, serviceInExternalEnv := range servicesInExternalEnv { + productServiceNames.Insert(serviceInExternalEnv.ServiceName) } var res []*Workload for _, workload := range workloads { - if productServiceNames.Has(fmt.Sprintf("%s/%s", workload.Type, workload.Name)) { + if productServiceNames.Has(workload.Name) { res = append(res, workload) } } return res }, } + + // for helm service, only show deploys/stss created by zadig + if projectInfo.ProductFeature != nil && projectInfo.ProductFeature.DeployType == setting.HelmDeployType { + filterArray = append(filterArray, func(workloads []*Workload) []*Workload { + releaseNameMap, err := GetReleaseNameToServiceNameMap(productInfo) + if err != nil { + log.Errorf("failed to generate relase map for product: %s:%s", productInfo.ProductName, productInfo.EnvName) + return workloads + } + + var res []*Workload + for _, workload := range workloads { + if len(workload.Annotation) == 0 { + continue + } + releaseName := workload.Annotation[setting.HelmReleaseNameAnnotation] + if _, ok := releaseNameMap[releaseName]; ok { + res = append(res, workload) + } + } + return res + }) + } + if filter != "" { filterArray = append(filterArray, func(workloads []*Workload) []*Workload { data, err := jsonutil.ToJSON(filter) if err != nil { - log.Warnf("Invalid filter, err: %s", err) + log.Errorf("Invalid filter, err: %s", err) return workloads } f := &workloadFilter{} if err = json.Unmarshal(data, f); err != nil { - log.Warnf("Invalid filter, err: %s", err) + log.Errorf("Invalid filter, err: %s", err) return workloads } @@ -171,59 +289,140 @@ func ListWorkloadsInEnv(envName, productName, filter string, perPage, page int, if err != nil { return count, resp, err } - fillServiceDisplayName(resp, productInfo) + + fillServiceInfo(resp, productInfo) return count, resp, nil } type FilterFunc func(services []*Workload) []*Workload type workloadFilter struct { - Name string `json:"name"` + Name string `json:"name"` + ServiceName string `json:"serviceName"` + ServiceNameList sets.String `json:"-"` +} + +type wfAlias workloadFilter + +func (f *workloadFilter) UnmarshalJSON(data []byte) error { + aliasData := &wfAlias{} + if err := json.Unmarshal(data, aliasData); err != nil { + return err + } + f.Name = aliasData.Name + f.ServiceName = aliasData.ServiceName + if f.ServiceName != "*" { + serviceNames := strings.Split(f.ServiceName, "|") + f.ServiceNameList = sets.NewString(serviceNames...) + } + return nil } func (f *workloadFilter) Match(workload *Workload) bool { - return strings.Contains(workload.Name, f.Name) + if len(f.Name) > 0 { + if !strings.Contains(workload.Name, f.Name) { + return false + } + } + if len(f.ServiceNameList) > 0 { + if !f.ServiceNameList.Has(workload.ServiceName) { + return false + } + } + return true } type Workload struct { - EnvName string `json:"env_name"` - Name string `json:"name"` - Type string `json:"type"` - ProductName string `json:"product_name"` - Spec corev1.ServiceSpec `json:"-"` - Images []string `json:"-"` - Ready bool + EnvName string `json:"env_name"` + Name string `json:"name"` + Type string `json:"type"` + ProductName string `json:"product_name"` + Spec corev1.PodTemplateSpec `json:"-"` + Images []string `json:"-"` + Ready bool `json:"ready"` + Annotation map[string]string `json:"-"` + ServiceName string `json:"service_name"` //serviceName refers to the service defines in zadig +} + +// fillServiceName set service name defined in zadig to workloads, this would be helpful for helm release view +// TODO optimize this function, use release.template to filter related resources +func fillServiceName(envName, productName string, workloads []*Workload) error { + if len(envName) == 0 { + return nil + } + productInfo, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: productName, + EnvName: envName, + }) + if err != nil { + return err + } + if productInfo.Source != setting.SourceFromHelm { + return nil + } + releaseNameMap, err := GetReleaseNameToServiceNameMap(productInfo) + if err != nil { + return err + } + for _, wl := range workloads { + if chartRelease, ok := wl.Annotation[setting.HelmReleaseNameAnnotation]; ok { + wl.ServiceName = releaseNameMap[chartRelease] + } + } + return nil } func ListWorkloads(envName, clusterID, namespace, productName string, perPage, page int, log *zap.SugaredLogger, filter ...FilterFunc) (int, []*ServiceResp, error) { - log.Infof("Start to list workloads in namespace %s", namespace) var resp = make([]*ServiceResp, 0) - kubeClient, err := kube.GetKubeClient(clusterID) + cls, err := kubeclient.GetKubeClientSet(config.HubServerAddress(), clusterID) + if err != nil { + log.Errorf("[%s][%s] error: %v", envName, namespace, err) + return 0, resp, e.ErrListGroups.AddDesc(err.Error()) + } + informer, err := informer.NewInformer(clusterID, namespace, cls) if err != nil { log.Errorf("[%s][%s] error: %v", envName, namespace, err) return 0, resp, e.ErrListGroups.AddDesc(err.Error()) } - var workLoads []*Workload - listDeployments, err := getter.ListDeployments(namespace, nil, kubeClient) + listDeployments, err := getter.ListDeploymentsWithCache(nil, informer) if err != nil { log.Errorf("[%s][%s] create product record error: %v", envName, namespace, err) return 0, resp, e.ErrListGroups.AddDesc(err.Error()) } + for _, v := range listDeployments { - workLoads = append(workLoads, &Workload{Name: v.Name, Spec: corev1.ServiceSpec{Selector: v.Spec.Selector.MatchLabels}, Type: setting.Deployment, Images: wrapper.Deployment(v).ImageInfos(), Ready: wrapper.Deployment(v).Ready()}) + workLoads = append(workLoads, &Workload{ + Name: v.Name, + Spec: v.Spec.Template, + Type: setting.Deployment, + Images: wrapper.Deployment(v).ImageInfos(), + Ready: wrapper.Deployment(v).Ready(), + Annotation: v.Annotations, + }) } - statefulSets, err := getter.ListStatefulSets(namespace, nil, kubeClient) + statefulSets, err := getter.ListStatefulSetsWithCache(nil, informer) if err != nil { log.Errorf("[%s][%s] create product record error: %v", envName, namespace, err) return 0, resp, e.ErrListGroups.AddDesc(err.Error()) } for _, v := range statefulSets { - workLoads = append(workLoads, &Workload{Name: v.Name, Spec: corev1.ServiceSpec{Selector: v.Spec.Selector.MatchLabels}, Type: setting.StatefulSet, Images: wrapper.StatefulSet(v).ImageInfos(), Ready: wrapper.StatefulSet(v).Ready()}) + workLoads = append(workLoads, &Workload{ + Name: v.Name, + Spec: v.Spec.Template, + Type: setting.StatefulSet, + Images: wrapper.StatefulSet(v).ImageInfos(), + Ready: wrapper.StatefulSet(v).Ready(), + Annotation: v.Annotations, + }) } - log.Debugf("Found %d workloads in total", len(workLoads)) + err = fillServiceName(envName, productName, workLoads) + // err of getting service name should not block the return of workloads + if err != nil { + log.Warnf("failed to set service name for workloads, error: %s", err) + } // 对于workload过滤 for _, f := range filter { @@ -232,8 +431,6 @@ func ListWorkloads(envName, clusterID, namespace, productName string, perPage, p count := len(workLoads) - log.Debugf("%d matching workloads left after filtering", count) - //将获取到的所有服务按照名称进行排序 sort.SliceStable(workLoads, func(i, j int) bool { return workLoads[i].Name < workLoads[j].Name }) @@ -249,6 +446,38 @@ func ListWorkloads(envName, clusterID, namespace, productName string, perPage, p } } + hostInfos := make([]resource.HostInfo, 0) + version, err := cls.Discovery().ServerVersion() + if err != nil { + log.Errorf("Failed to get server version info for cluster: %s, the error is: %s", clusterID, err) + return 0, nil, err + } + if kubeclient.VersionLessThan122(version) { + ingresses, err := getter.ListExtensionsV1Beta1Ingresses(nil, informer) + if err == nil { + for _, ingress := range ingresses { + hostInfos = append(hostInfos, wrapper.Ingress(ingress).HostInfo()...) + } + } else { + log.Warnf("Failed to list ingresses, the error is: %s", err) + } + } else { + ingresses, err := getter.ListNetworkingV1Ingress(nil, informer) + if err == nil { + for _, ingress := range ingresses { + hostInfos = append(hostInfos, wrapper.GetIngressHostInfo(ingress)...) + } + } else { + log.Warnf("Failed to list ingresses, the error is: %s", err) + } + } + + // get all services + allServices, err := getter.ListServicesWithCache(nil, informer) + if err != nil { + log.Errorf("[%s][%s] list service error: %s", envName, namespace, err) + } + for _, workload := range workLoads { tmpProductName := workload.ProductName if tmpProductName == "" && productName != "" { @@ -264,14 +493,136 @@ func ListWorkloads(envName, clusterID, namespace, productName string, perPage, p Ready: setting.PodReady, Status: setting.PodRunning, } - if !workload.Ready { - productRespInfo.Status = setting.PodUnstable - productRespInfo.Ready = setting.PodNotReady + + selector := labels.SelectorFromSet(labels.Set(workload.Spec.Labels)) + // Note: In some scenarios, such as environment sharing, there may be more containers in Pod than workload. + // We call GetSelectedPodsInfo to get the status and readiness to keep same logic with k8s projects + productRespInfo.Status, productRespInfo.Ready, productRespInfo.Images = kube.GetSelectedPodsInfo(selector, informer, log) + + productRespInfo.Ingress = &IngressInfo{ + HostInfo: findServiceFromIngress(hostInfos, workload, allServices), } + resp = append(resp, productRespInfo) } - log.Infof("Finish to list workloads in namespace %s", namespace) - return count, resp, nil } + +func findServiceFromIngress(hostInfos []resource.HostInfo, currentWorkload *Workload, allServices []*corev1.Service) []resource.HostInfo { + if len(allServices) == 0 || len(hostInfos) == 0 { + return []resource.HostInfo{} + } + serviceName := "" + podLabels := labels.Set(currentWorkload.Spec.Labels) + for _, svc := range allServices { + if len(svc.Spec.Selector) == 0 { + continue + } + if labels.SelectorFromValidatedSet(svc.Spec.Selector).Matches(podLabels) { + serviceName = svc.Name + break + } + } + if serviceName == "" { + return []resource.HostInfo{} + } + + resp := make([]resource.HostInfo, 0) + for _, hostInfo := range hostInfos { + for _, backend := range hostInfo.Backends { + if backend.ServiceName == serviceName { + resp = append(resp, hostInfo) + break + } + } + } + return resp +} + +func GetProductUsedTemplateSvcs(prod *models.Product) ([]*models.Service, error) { + // filter releases, only list releases deployed by zadig + productName, envName, serviceMap := prod.ProductName, prod.EnvName, prod.GetServiceMap() + if len(serviceMap) == 0 { + return nil, nil + } + listOpt := &commonrepo.SvcRevisionListOption{ + ProductName: prod.ProductName, + ServiceRevisions: make([]*commonrepo.ServiceRevision, 0), + } + for _, productSvc := range serviceMap { + listOpt.ServiceRevisions = append(listOpt.ServiceRevisions, &commonrepo.ServiceRevision{ + ServiceName: productSvc.ServiceName, + Revision: productSvc.Revision, + }) + } + templateServices, err := commonrepo.NewServiceColl().ListServicesWithSRevision(listOpt) + if err != nil { + return nil, fmt.Errorf("failed to list template services for pruduct: %s:%s, err: %s", productName, envName, err) + } + return templateServices, err +} + +// GetReleaseNameToServiceNameMap generates mapping relationship: releaseName=>serviceName +func GetReleaseNameToServiceNameMap(prod *models.Product) (map[string]string, error) { + productName, envName := prod.ProductName, prod.EnvName + templateServices, err := GetProductUsedTemplateSvcs(prod) + if err != nil { + return nil, err + } + // map[ReleaseName] => serviceName + releaseNameMap := make(map[string]string) + for _, svcInfo := range templateServices { + releaseNameMap[util.GeneReleaseName(svcInfo.GetReleaseNaming(), productName, prod.Namespace, envName, svcInfo.ServiceName)] = svcInfo.ServiceName + } + return releaseNameMap, nil +} + +// GetServiceNameToReleaseNameMap generates mapping relationship: serviceName=>releaseName +func GetServiceNameToReleaseNameMap(prod *models.Product) (map[string]string, error) { + productName, envName := prod.ProductName, prod.EnvName + templateServices, err := GetProductUsedTemplateSvcs(prod) + if err != nil { + return nil, err + } + // map[serviceName] => ReleaseName + releaseNameMap := make(map[string]string) + for _, svcInfo := range templateServices { + releaseNameMap[svcInfo.ServiceName] = util.GeneReleaseName(svcInfo.GetReleaseNaming(), productName, prod.Namespace, envName, svcInfo.ServiceName) + } + return releaseNameMap, nil +} + +// GetHelmServiceName get service name from annotations of resources deployed by helm +// resType currently only support Deployment and StatefulSet +// this function needs to be optimized +func GetHelmServiceName(prod *models.Product, resType, resName string, kubeClient client.Client) (string, error) { + res := &unstructured.Unstructured{} + namespace := prod.Namespace + + nameMap, err := GetReleaseNameToServiceNameMap(prod) + if err != nil { + return "", err + } + + res.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "apps", + Version: "v1", + Kind: resType, + }) + found, err := getter.GetResourceInCache(namespace, resName, res, kubeClient) + if err != nil { + return "", fmt.Errorf("failed to find resource %s, type %s, err %s", resName, resType, err.Error()) + } + if !found { + return "", fmt.Errorf("failed to find resource %s, type %s", resName, resType) + } + annotation := res.GetAnnotations() + if len(annotation) > 0 { + releaseName := annotation[setting.HelmReleaseNameAnnotation] + if serviceName, ok := nameMap[releaseName]; ok { + return serviceName, nil + } + } + return "", fmt.Errorf("failed to get annotation from resource %s, type %s", resName, resType) +} diff --git a/pkg/microservice/aslan/core/common/service/fs/file.go b/pkg/microservice/aslan/core/common/service/fs/file.go index c5317ae9f7ea15a66b02a5796c771bf9853363f1..7d513cd40de40d07b763633b749b5c229f2da54b 100644 --- a/pkg/microservice/aslan/core/common/service/fs/file.go +++ b/pkg/microservice/aslan/core/common/service/fs/file.go @@ -20,15 +20,17 @@ import ( "io/fs" "os" + "github.com/otiai10/copy" "go.uber.org/zap" "k8s.io/apimachinery/pkg/util/wait" + "github.com/koderover/zadig/pkg/setting" fsutil "github.com/koderover/zadig/pkg/util/fs" ) // PreloadFiles downloads a tarball from object storage and extracts it to a local path for further usage. // It happens only if files do not exist in local disk. -func PreloadFiles(name, localBase, s3Base string, logger *zap.SugaredLogger) error { +func PreloadFiles(name, localBase, s3Base, source string, logger *zap.SugaredLogger) error { ok, err := fsutil.DirExists(localBase) if err != nil { logger.Errorf("Failed to check if dir %s is exiting, err: %s", localBase, err) @@ -38,16 +40,29 @@ func PreloadFiles(name, localBase, s3Base string, logger *zap.SugaredLogger) err return nil } - if err = DownloadAndExtractFilesFromS3(name, localBase, s3Base, logger); err != nil { - logger.Errorf("Failed to download files from s3, err: %s", err) - return err + switch source { + case setting.SourceFromGerrit: + if err = DownloadAndCopyFilesFromGerrit(name, localBase, logger); err != nil { + logger.Errorf("Failed to download files from s3, err: %s", err) + return err + } + case setting.SourceFromGitee: + if err = DownloadAndCopyFilesFromGitee(name, localBase, logger); err != nil { + logger.Errorf("Failed to download files from s3, err: %s", err) + return err + } + default: + if err = DownloadAndExtractFilesFromS3(name, localBase, s3Base, logger); err != nil { + logger.Errorf("Failed to download files from s3, err: %s", err) + return err + } } return nil } // SaveAndUploadFiles saves a tree of files to local disk, at the same time, archives them and uploads to object storage. -func SaveAndUploadFiles(fileTree fs.FS, name, localBase, s3Base string, logger *zap.SugaredLogger) error { +func SaveAndUploadFiles(fileTree fs.FS, names []string, localBase, s3Base string, logger *zap.SugaredLogger) error { var wg wait.Group var err error @@ -58,8 +73,9 @@ func SaveAndUploadFiles(fileTree fs.FS, name, localBase, s3Base string, logger * err = err1 } }) + wg.Start(func() { - err2 := ArchiveAndUploadFilesToS3(fileTree, name, s3Base, logger) + err2 := ArchiveAndUploadFilesToS3(fileTree, names, s3Base, logger) if err2 != nil { logger.Errorf("Failed to upload files to s3, err: %s", err2) err = err2 @@ -71,6 +87,35 @@ func SaveAndUploadFiles(fileTree fs.FS, name, localBase, s3Base string, logger * return err } +// CopyAndUploadFiles copy a tree of files to other dir, at the same time, archives them and uploads to object storage. +func CopyAndUploadFiles(names []string, localBase, s3Base, currentChartPath string, logger *zap.SugaredLogger) error { + var wg wait.Group + var err error + + wg.Start(func() { + copyErr := copy.Copy(currentChartPath, localBase) + if copyErr != nil { + logger.Errorf("failed to copy chart info, err %s", copyErr) + err = copyErr + } + }) + + wg.Start(func() { + fileTree := os.DirFS(currentChartPath) + if s3Base == "" { + return + } + err2 := ArchiveAndUploadFilesToS3(fileTree, names, s3Base, logger) + if err2 != nil { + logger.Errorf("Failed to upload files to s3, err: %s", err2) + err = err2 + } + }) + + wg.Wait() + return err +} + func saveInMemoryFilesToDisk(fileTree fs.FS, root string, logger *zap.SugaredLogger) error { ok, err := fsutil.DirExists(root) if err != nil { diff --git a/pkg/microservice/aslan/core/common/service/fs/git.go b/pkg/microservice/aslan/core/common/service/fs/git.go index 12b5c9003288bdc06e54e452a43895d7a61861b8..2bcdbb0bd2f08ef0b58ec3355f58fbce570e4176 100644 --- a/pkg/microservice/aslan/core/common/service/fs/git.go +++ b/pkg/microservice/aslan/core/common/service/fs/git.go @@ -27,27 +27,35 @@ import ( githubservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/github" gitlabservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/gitlab" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/codehost" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" "github.com/koderover/zadig/pkg/tool/log" ) type DownloadFromSourceArgs struct { CodehostID int `json:"codeHostID"` Owner string `json:"owner"` + Namespace string `json:"namespace"` Repo string `json:"repo"` Path string `json:"path"` Branch string `json:"branch"` RepoLink string `json:"repoLink"` } +func (args *DownloadFromSourceArgs) GetNamespace() string { + if len(args.Namespace) > 0 { + return args.Namespace + } + return args.Owner +} + func DownloadFileFromSource(args *DownloadFromSourceArgs) ([]byte, error) { getter, err := treeGetter(args.RepoLink, args.CodehostID) if err != nil { log.Errorf("Failed to get tree getter, err: %s", err) return nil, err } - - return getter.GetFileContent(args.Owner, args.Repo, args.Path, args.Branch) + owner := args.GetNamespace() + return getter.GetFileContent(owner, args.Repo, args.Path, args.Branch) } func DownloadFilesFromSource(args *DownloadFromSourceArgs, rootNameGetter func(afero.Fs) (string, error)) (fs.FS, error) { @@ -56,8 +64,11 @@ func DownloadFilesFromSource(args *DownloadFromSourceArgs, rootNameGetter func(a log.Errorf("Failed to get tree getter, err: %s", err) return nil, err } - - chartTree, err := getter.GetTreeContents(args.Owner, args.Repo, args.Path, args.Branch) + owner := args.Namespace + if owner == "" { + owner = args.Owner + } + chartTree, err := getter.GetTreeContents(owner, args.Repo, args.Path, args.Branch) if err != nil { log.Errorf("Failed to get tree contents for service %+v, err: %s", args, err) return nil, err @@ -94,14 +105,15 @@ type TreeGetter interface { GetTreeContents(owner, repo, path, branch string) (afero.Fs, error) GetFileContent(owner, repo, path, branch string) ([]byte, error) GetTree(owner, repo, path, branch string) ([]*git.TreeNode, error) + GetYAMLContents(owner, repo, path, branch string, isDir, split bool) ([]string, error) } func GetPublicTreeGetter(repoLink string) (TreeGetter, error) { - return githubservice.NewClient("", config.ProxyHTTPSAddr()), nil + return githubservice.NewClient("", config.ProxyHTTPSAddr(), true), nil } func GetTreeGetter(codeHostID int) (TreeGetter, error) { - ch, err := codehost.GetCodeHostInfoByID(codeHostID) + ch, err := systemconfig.New().GetCodeHost(codeHostID) if err != nil { log.Errorf("Failed to get codeHost by id %d, err: %s", codeHostID, err) return nil, err @@ -109,9 +121,9 @@ func GetTreeGetter(codeHostID int) (TreeGetter, error) { switch ch.Type { case setting.SourceFromGithub: - return githubservice.NewClient(ch.AccessToken, config.ProxyHTTPSAddr()), nil + return githubservice.NewClient(ch.AccessToken, config.ProxyHTTPSAddr(), ch.EnableProxy), nil case setting.SourceFromGitlab: - return gitlabservice.NewClient(ch.Address, ch.AccessToken) + return gitlabservice.NewClient(ch.ID, ch.Address, ch.AccessToken, config.ProxyHTTPSAddr(), ch.EnableProxy) default: // should not have happened here log.DPanicf("invalid source: %s", ch.Type) diff --git a/pkg/microservice/aslan/core/common/service/fs/s3.go b/pkg/microservice/aslan/core/common/service/fs/s3.go index a2e0eb7a043a553e89db5e75ee4578fc13ae1453..bdcba89e3d6c9edd3715a542e5a2108a15264a1c 100644 --- a/pkg/microservice/aslan/core/common/service/fs/s3.go +++ b/pkg/microservice/aslan/core/common/service/fs/s3.go @@ -20,17 +20,46 @@ import ( "fmt" "io/fs" "os" + "path" "path/filepath" "go.uber.org/zap" + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/command" s3service "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" s3tool "github.com/koderover/zadig/pkg/tool/s3" fsutil "github.com/koderover/zadig/pkg/util/fs" ) -func ArchiveAndUploadFilesToS3(fileTree fs.FS, name, s3Base string, logger *zap.SugaredLogger) error { +func ArchiveAndUploadFilesToSpecifiedS3(fileTree fs.FS, names []string, s3Base, s3Id string, logger *zap.SugaredLogger) error { + s3Storage, err := s3service.FindS3ById(s3Id) + if err != nil { + logger.Errorf("Failed to find default s3, err:%v", err) + return err + } + return archiveAndUploadFiles(fileTree, names, s3Base, s3Storage, logger) +} + +func ArchiveAndUploadFilesToS3(fileTree fs.FS, names []string, s3Base string, logger *zap.SugaredLogger) error { + s3Storage, err := s3service.FindDefaultS3() + if err != nil { + logger.Errorf("Failed to find default s3, err:%v", err) + return err + } + return archiveAndUploadFiles(fileTree, names, s3Base, s3Storage, logger) +} + +// archiveAndUploadFiles archive local files and upload to default s3 storage +// if multiple names appointed, s3storage.copy will be used to handle extra names +func archiveAndUploadFiles(fileTree fs.FS, names []string, s3Base string, s3Storage *s3service.S3, logger *zap.SugaredLogger) error { + if len(names) == 0 { + return fmt.Errorf("names not appointed") + } + tmpDir, err := os.MkdirTemp("", "") if err != nil { logger.Errorf("Failed to create temp dir, err: %s", err) @@ -38,17 +67,16 @@ func ArchiveAndUploadFilesToS3(fileTree fs.FS, name, s3Base string, logger *zap. } defer os.RemoveAll(tmpDir) + name := names[0] + copies := names[1:] + tarball := fmt.Sprintf("%s.tar.gz", name) localPath := filepath.Join(tmpDir, tarball) if err = fsutil.Tar(fileTree, localPath); err != nil { logger.Errorf("Failed to archive tarball %s, err: %s", localPath, err) return err } - s3Storage, err := s3service.FindDefaultS3() - if err != nil { - logger.Errorf("Failed to find default s3, err:%v", err) - return err - } + forcedPathStyle := true if s3Storage.Provider == setting.ProviderSourceAli { forcedPathStyle = false @@ -64,6 +92,16 @@ func ArchiveAndUploadFilesToS3(fileTree fs.FS, name, s3Base string, logger *zap. return err } + // copy file to avoid duplicated file transfer + for _, copyName := range copies { + targetPath := filepath.Join(s3Storage.Subfolder, s3Base, fmt.Sprintf("%s.tar.gz", copyName)) + err = client.CopyObject(s3Storage.Bucket, s3Path, targetPath) + if err != nil { + logger.Errorf("Failed to copy object from %s to %s", s3Path, targetPath) + return err + } + } + return nil } @@ -73,13 +111,11 @@ func DownloadAndExtractFilesFromS3(name, localBase, s3Base string, logger *zap.S logger.Errorf("Failed to create temp dir, err: %s", err) } defer os.RemoveAll(tmpDir) - s3, err := s3service.FindDefaultS3() if err != nil { logger.Errorf("Failed to find default s3, err: %s", err) return err } - tarball := fmt.Sprintf("%s.tar.gz", name) localPath := filepath.Join(tmpDir, tarball) s3Path := filepath.Join(s3.Subfolder, s3Base, tarball) @@ -88,32 +124,39 @@ func DownloadAndExtractFilesFromS3(name, localBase, s3Base string, logger *zap.S if s3.Provider == setting.ProviderSourceAli { forcedPathStyle = false } + client, err := s3tool.NewClient(s3.Endpoint, s3.Ak, s3.Sk, s3.Insecure, forcedPathStyle) if err != nil { logger.Errorf("Failed to create s3 client, err: %s", err) return err } + if err = client.Download(s3.Bucket, s3Path, localPath); err != nil { logger.Errorf("Failed to download file from s3, err: %s", err) return err } + if err = fsutil.Untar(localPath, localBase); err != nil { logger.Errorf("Failed to extract tarball %s, err: %s", localPath, err) return err } - return nil } -func DeleteArchivedFileFromS3(name, s3Base string, logger *zap.SugaredLogger) error { +func DeleteArchivedFileFromS3(names []string, s3Base string, logger *zap.SugaredLogger) error { s3, err := s3service.FindDefaultS3() if err != nil { logger.Errorf("Failed to find default s3, err: %s", err) return err } - tarball := fmt.Sprintf("%s.tar.gz", name) - s3Path := filepath.Join(s3.Subfolder, s3Base, tarball) + s3PathList := make([]string, 0, len(names)) + for _, name := range s3PathList { + tarball := fmt.Sprintf("%s.tar.gz", name) + s3Path := filepath.Join(s3.Subfolder, s3Base, tarball) + s3PathList = append(s3PathList, s3Path) + } + forcedPathStyle := true if s3.Provider == setting.ProviderSourceAli { forcedPathStyle = false @@ -124,5 +167,57 @@ func DeleteArchivedFileFromS3(name, s3Base string, logger *zap.SugaredLogger) er return err } - return client.DeleteObjects(s3.Bucket, []string{s3Path}) + return client.DeleteObjects(s3.Bucket, s3PathList) +} + +func DownloadAndCopyFilesFromGerrit(name, localBase string, logger *zap.SugaredLogger) error { + chartTemplate, err := mongodb.NewChartColl().Get(name) + base := path.Join(config.S3StoragePath(), chartTemplate.Repo) + if err := os.RemoveAll(base); err != nil { + logger.Warnf("Failed to remove dir, err:%s", err) + } + detail, err := systemconfig.New().GetCodeHost(chartTemplate.CodeHostID) + if err != nil { + logger.Errorf("Failed to GetCodehostDetail, err:%s", err) + return err + } + err = command.RunGitCmds(detail, "default", "default", chartTemplate.Repo, chartTemplate.Branch, "origin") + if err != nil { + logger.Errorf("Failed to runGitCmds, err:%s", err) + return err + } + + if err := CopyAndUploadFiles([]string{}, path.Join(localBase, path.Base(chartTemplate.Path)), "", path.Join(base, chartTemplate.Path), logger); err != nil { + logger.Errorf("Failed to copy files for helm chart template %s, error: %s", name, err) + return err + } + return nil +} + +func DownloadAndCopyFilesFromGitee(name, localBase string, logger *zap.SugaredLogger) error { + chartTemplate, err := mongodb.NewChartColl().Get(name) + if err != nil { + logger.Errorf("Failed to get chart template, err:%s", err) + return err + } + base := path.Join(config.S3StoragePath(), chartTemplate.Repo) + if err := os.RemoveAll(base); err != nil { + logger.Warnf("Failed to remove dir, err:%s", err) + } + detail, err := systemconfig.New().GetCodeHost(chartTemplate.CodeHostID) + if err != nil { + logger.Errorf("Failed to get codehost detail, err:%s", err) + return err + } + err = command.RunGitCmds(detail, chartTemplate.Owner, chartTemplate.GetNamespace(), chartTemplate.Repo, chartTemplate.Branch, "origin") + if err != nil { + logger.Errorf("Failed to runGitCmds, err:%s", err) + return err + } + + if err := CopyAndUploadFiles([]string{}, path.Join(localBase, path.Base(chartTemplate.Path)), "", path.Join(base, chartTemplate.Path), logger); err != nil { + logger.Errorf("Failed to copy files for helm chart template %s, error: %s", name, err) + return err + } + return nil } diff --git a/pkg/microservice/aslan/core/common/service/gerrit/gerrit.go b/pkg/microservice/aslan/core/common/service/gerrit/gerrit.go index 5706817753bedf002233420d38b0cf159a57f013..516d4e17c28fb29ff0bcd0d2bf3305ab9445b8f3 100644 --- a/pkg/microservice/aslan/core/common/service/gerrit/gerrit.go +++ b/pkg/microservice/aslan/core/common/service/gerrit/gerrit.go @@ -26,7 +26,7 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" - "github.com/koderover/zadig/pkg/shared/codehost" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" "github.com/koderover/zadig/pkg/tool/gerrit" ) @@ -37,7 +37,7 @@ func DeleteGerritWebhook(workflow *models.Workflow, log *zap.SugaredLogger) erro continue } - detail, err := codehost.GetCodeHostInfoByID(workflowWebhook.MainRepo.CodehostID) + detail, err := systemconfig.New().GetCodeHost(workflowWebhook.MainRepo.CodehostID) if err != nil { log.Errorf("DeleteGerritWebhook GetCodehostDetail err:%v", err) continue diff --git a/pkg/microservice/aslan/core/common/service/git/hook.go b/pkg/microservice/aslan/core/common/service/git/hook.go index dcff7666cf3b146f73d2a70d1a7003741887aea8..08770cc51ad020970fc30268e35befc99efe739d 100644 --- a/pkg/microservice/aslan/core/common/service/git/hook.go +++ b/pkg/microservice/aslan/core/common/service/git/hook.go @@ -17,26 +17,14 @@ limitations under the License. package git import ( - "sync" + "crypto/sha256" + "encoding/hex" - "github.com/koderover/zadig/pkg/microservice/aslan/config" - "github.com/koderover/zadig/pkg/shared/poetry" - "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/config" ) -var once sync.Once -var secret string - func GetHookSecret() string { - once.Do(func() { - poetryClient := poetry.New(config.PoetryAPIServer(), config.PoetryAPIRootKey()) - org, err := poetryClient.GetOrganization(poetry.DefaultOrganization) - if err != nil { - log.Errorf("failed to find default organization: %v", err) - secret = "--impossible-token--" - } - secret = org.Token - }) - - return secret + hash := sha256.Sum256([]byte(config.SecretKey())) + + return hex.EncodeToString(hash[:]) } diff --git a/pkg/microservice/aslan/core/common/service/gitee/client.go b/pkg/microservice/aslan/core/common/service/gitee/client.go new file mode 100644 index 0000000000000000000000000000000000000000..264a35ccb775da6be066bfdff73c6266684520c0 --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/gitee/client.go @@ -0,0 +1,34 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gitee + +import ( + "github.com/koderover/zadig/pkg/tool/gitee" +) + +type Client struct { + *gitee.Client + AccessToken string +} + +func NewClient(id int, accessToken, proxyAddress string, enableProxy bool) *Client { + client := gitee.NewClient(id, accessToken, proxyAddress, enableProxy) + return &Client{ + Client: client, + AccessToken: accessToken, + } +} diff --git a/pkg/microservice/aslan/core/common/service/gitee/comment.go b/pkg/microservice/aslan/core/common/service/gitee/comment.go new file mode 100644 index 0000000000000000000000000000000000000000..5d9e5b82bfa7b5f6816483ed73380aa0ccd08fba --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/gitee/comment.go @@ -0,0 +1,31 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gitee + +import ( + "context" + + "gitee.com/openeuler/go-gitee/gitee" +) + +func (c *Client) CreateMergeRequestComment(ctx context.Context, owner, repo string, number int32, pullRequestCommentPostParam gitee.PullRequestCommentPostParam) (gitee.PullRequestComments, error) { + return c.Client.CreateMergeRequestComment(ctx, owner, repo, number, pullRequestCommentPostParam) +} + +func (c *Client) UpdateMergeRequestComment(ctx context.Context, owner, repo string, id int32, pullRequestCommentPatchParam gitee.PullRequestCommentPatchParam) error { + return c.Client.UpdateMergeRequestComment(ctx, owner, repo, id, pullRequestCommentPatchParam) +} diff --git a/pkg/microservice/aslan/core/common/service/ilyshin/webhook.go b/pkg/microservice/aslan/core/common/service/gitee/webhook.go similarity index 66% rename from pkg/microservice/aslan/core/common/service/ilyshin/webhook.go rename to pkg/microservice/aslan/core/common/service/gitee/webhook.go index 4a8dd644bcf5066f50e214c089e5f84e9dcd6bff..fe90ad6c5a88372e1421d1d58ffd5ec1f54bdc1f 100644 --- a/pkg/microservice/aslan/core/common/service/ilyshin/webhook.go +++ b/pkg/microservice/aslan/core/common/service/gitee/webhook.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The KodeRover Authors. +Copyright 2022 The KodeRover Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,25 +14,33 @@ See the License for the specific language governing permissions and limitations under the License. */ -package ilyshin +package gitee import ( + "context" "strconv" "github.com/koderover/zadig/pkg/microservice/aslan/config" gitservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/git" + "github.com/koderover/zadig/pkg/tool/git" ) func (c *Client) CreateWebHook(owner, repo string) (string, error) { - projectHook, err := c.AddProjectHook(owner, repo, config.WebHookURL(), gitservice.GetHookSecret()) + hook, err := c.CreateHook(c.AccessToken, owner, repo, &git.Hook{ + URL: config.WebHookURL(), + Secret: gitservice.GetHookSecret(), + }) + if err != nil { + return "", err + } - return strconv.Itoa(projectHook.ID), err + return strconv.Itoa(int(hook.ID)), nil } func (c *Client) DeleteWebHook(owner, repo, hookID string) error { - hookIDInt, err := strconv.Atoi(hookID) + hookIDInt, err := strconv.ParseInt(hookID, 10, 64) if err != nil { return err } - return c.DeleteProjectHook(owner, repo, hookIDInt) + return c.DeleteHook(context.TODO(), owner, repo, hookIDInt) } diff --git a/pkg/microservice/aslan/core/common/service/github/client.go b/pkg/microservice/aslan/core/common/service/github/client.go index fc33e1db22a85f52419c8a1a4c53ba4e03cf7ca1..522b851d77869d48d3ee2c85c443c96deab437d5 100644 --- a/pkg/microservice/aslan/core/common/service/github/client.go +++ b/pkg/microservice/aslan/core/common/service/github/client.go @@ -27,9 +27,15 @@ type Client struct { // InstallationToken string } -func NewClient(accessToken, proxyAddress string) *Client { +func NewClient(accessToken, proxyAddress string, enableProxy bool) *Client { + cfg := &github.Config{ + AccessToken: accessToken, + } + if enableProxy { + cfg.Proxy = proxyAddress + } return &Client{ - Client: github.NewClient(&github.Config{AccessToken: accessToken, Proxy: proxyAddress}), + Client: github.NewClient(cfg), } } diff --git a/pkg/microservice/aslan/core/common/service/github/repository.go b/pkg/microservice/aslan/core/common/service/github/repository.go index 06134f34a29bfabe42f8fcbae5e540ec95af9887..8745b2416a7130c8cdf3430ffe215b88dda92df4 100644 --- a/pkg/microservice/aslan/core/common/service/github/repository.go +++ b/pkg/microservice/aslan/core/common/service/github/repository.go @@ -62,6 +62,5 @@ func (c *Client) GetFileContent(owner, repo, path, branch string) ([]byte, error } res, err := fileContent.GetContent() - return []byte(res), err } diff --git a/pkg/microservice/aslan/core/common/service/github/status.go b/pkg/microservice/aslan/core/common/service/github/status.go index 501960b92c05852cc7b1ee5c70fd21b5a918f51f..18b8edb2e0487e04d2381bb7965556f1ddf5a23c 100644 --- a/pkg/microservice/aslan/core/common/service/github/status.go +++ b/pkg/microservice/aslan/core/common/service/github/status.go @@ -47,7 +47,7 @@ type StatusOptions struct { } func (c *Client) UpdateCheckStatus(opt *StatusOptions) error { - sc := setting.ProductName + sc := setting.ProductName + "/" + opt.PipeName _, err := c.CreateStatus( context.TODO(), opt.Owner, opt.Repo, opt.Ref, &github.RepoStatus{ diff --git a/pkg/microservice/aslan/core/common/service/github/webhook.go b/pkg/microservice/aslan/core/common/service/github/webhook.go index ebd172285ded81b45a63a43d50b5cd15bdadf038..645bf05d1e8277bc042366c69ab47dae4ca60eea 100644 --- a/pkg/microservice/aslan/core/common/service/github/webhook.go +++ b/pkg/microservice/aslan/core/common/service/github/webhook.go @@ -31,14 +31,30 @@ func (c *Client) CreateWebHook(owner, repo string) (string, error) { Secret: gitservice.GetHookSecret(), Events: []string{git.PushEvent, git.PullRequestEvent, git.BranchOrTagCreateEvent, git.CheckRunEvent}, }) + if err != nil { + return "", err + } - return strconv.Itoa(int(hook.GetID())), err + return strconv.Itoa(int(hook.GetID())), nil } func (c *Client) DeleteWebHook(owner, repo, hookID string) error { - hookIDInt, err := strconv.Atoi(hookID) + hookIDInt, err := strconv.ParseInt(hookID, 10, 64) + if err != nil { + return err + } + return c.DeleteHook(context.TODO(), owner, repo, hookIDInt) +} + +func (c *Client) RefreshWebHookSecret(secret, owner, repo, hookID string) error { + hookIDInt, err := strconv.ParseInt(hookID, 10, 64) if err != nil { return err } - return c.DeleteHook(context.TODO(), owner, repo, int64(hookIDInt)) + _, err = c.UpdateHook(context.TODO(), owner, repo, hookIDInt, &git.Hook{ + URL: config.WebHookURL(), + Secret: secret, + }) + + return err } diff --git a/pkg/microservice/aslan/core/common/service/gitlab/client.go b/pkg/microservice/aslan/core/common/service/gitlab/client.go index 4d5051e087db16939540f60ea03b0bf2e1d6679d..6f8326f73cdc516585d3d6c69d032ec4dfba2663 100644 --- a/pkg/microservice/aslan/core/common/service/gitlab/client.go +++ b/pkg/microservice/aslan/core/common/service/gitlab/client.go @@ -24,8 +24,8 @@ type Client struct { *gitlab.Client } -func NewClient(address, accessToken string) (*Client, error) { - c, err := gitlab.NewClient(address, accessToken) +func NewClient(id int, address, accessToken, proxyAddr string, enableProxy bool) (*Client, error) { + c, err := gitlab.NewClient(id, address, accessToken, proxyAddr, enableProxy) if err != nil { return nil, err } diff --git a/pkg/microservice/aslan/core/common/service/gitlab/webhook.go b/pkg/microservice/aslan/core/common/service/gitlab/webhook.go index 677f225e44317bd83c4aa4fe7462f2937f8554bd..f1a1e772e9cc11c00a26508ac9745c8505ecd18b 100644 --- a/pkg/microservice/aslan/core/common/service/gitlab/webhook.go +++ b/pkg/microservice/aslan/core/common/service/gitlab/webhook.go @@ -30,8 +30,11 @@ func (c *Client) CreateWebHook(owner, repo string) (string, error) { Secret: gitservice.GetHookSecret(), Events: []string{git.PushEvent, git.PullRequestEvent, git.BranchOrTagCreateEvent}, }) + if err != nil { + return "", err + } - return strconv.Itoa(projectHook.ID), err + return strconv.Itoa(projectHook.ID), nil } func (c *Client) DeleteWebHook(owner, repo, hookID string) error { @@ -41,3 +44,16 @@ func (c *Client) DeleteWebHook(owner, repo, hookID string) error { } return c.DeleteProjectHook(owner, repo, hookIDInt) } + +func (c *Client) RefreshWebHookSecret(secret, owner, repo, hookID string) error { + hookIDInt, err := strconv.Atoi(hookID) + if err != nil { + return err + } + _, err = c.UpdateProjectHook(owner, repo, hookIDInt, &git.Hook{ + URL: config.WebHookURL(), + Secret: secret, + }) + + return err +} diff --git a/pkg/microservice/aslan/core/common/service/helm.go b/pkg/microservice/aslan/core/common/service/helm.go index b905754fe29c30e961f92be3b4ecddfe1083886f..11639d1494ffc0c19bf92e936fe4cd068e6ae08a 100644 --- a/pkg/microservice/aslan/core/common/service/helm.go +++ b/pkg/microservice/aslan/core/common/service/helm.go @@ -18,25 +18,43 @@ package service import ( "io/fs" + "os" + "path" "github.com/27149chen/afero" "go.uber.org/zap" + "helm.sh/helm/v3/pkg/repo" "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/command" fsservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/fs" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" + "github.com/koderover/zadig/pkg/tool/crypto" "github.com/koderover/zadig/pkg/tool/log" fsutil "github.com/koderover/zadig/pkg/util/fs" ) -func ListHelmRepos(log *zap.SugaredLogger) ([]*commonmodels.HelmRepo, error) { +func ListHelmRepos(encryptedKey string, log *zap.SugaredLogger) ([]*commonmodels.HelmRepo, error) { + aesKey, err := GetAesKeyFromEncryptedKey(encryptedKey, log) + if err != nil { + log.Errorf("ListHelmRepos GetAesKeyFromEncryptedKey err:%v", err) + return nil, err + } helmRepos, err := commonrepo.NewHelmRepoColl().List() if err != nil { log.Errorf("ListHelmRepos err:%v", err) return []*commonmodels.HelmRepo{}, nil } - + for _, helmRepo := range helmRepos { + helmRepo.Password, err = crypto.AesEncryptByKey(helmRepo.Password, aesKey.PlainText) + if err != nil { + log.Errorf("ListHelmRepos AesEncryptByKey err:%v", err) + return nil, err + } + } return helmRepos, nil } @@ -55,25 +73,56 @@ func PreLoadServiceManifests(base string, svc *commonmodels.Service) error { } log.Warnf("Failed to download service from s3, err: %s", err) + switch svc.Source { + case setting.SourceFromGerrit: + return preLoadServiceManifestsFromGerrit(svc) + case setting.SourceFromGitee: + return preLoadServiceManifestsFromGitee(svc) + default: + return preLoadServiceManifestsFromSource(svc) + } +} - return preLoadServiceManifestsFromSource(svc) +func PreloadServiceManifestsByRevision(base string, svc *commonmodels.Service) error { + ok, err := fsutil.DirExists(base) + if err != nil { + log.Errorf("Failed to check if dir %s is exiting, err: %s", base, err) + return err + } + if ok { + return nil + } + + //download chart info by revision + serviceNameWithRevision := config.ServiceNameWithRevision(svc.ServiceName, svc.Revision) + s3Base := config.ObjectStorageServicePath(svc.ProductName, svc.ServiceName) + return fsservice.DownloadAndExtractFilesFromS3(serviceNameWithRevision, base, s3Base, log.SugaredLogger()) } func DownloadServiceManifests(base, projectName, serviceName string) error { s3Base := config.ObjectStorageServicePath(projectName, serviceName) + return fsservice.DownloadAndExtractFilesFromS3(serviceName, base, s3Base, log.SugaredLogger()) } -func SaveAndUploadService(projectName, serviceName string, fileTree fs.FS) error { +func SaveAndUploadService(projectName, serviceName string, copies []string, fileTree fs.FS) error { + localBase := config.LocalServicePath(projectName, serviceName) + s3Base := config.ObjectStorageServicePath(projectName, serviceName) + names := append([]string{serviceName}, copies...) + return fsservice.SaveAndUploadFiles(fileTree, names, localBase, s3Base, log.SugaredLogger()) +} + +func CopyAndUploadService(projectName, serviceName, currentChartPath string, copies []string) error { localBase := config.LocalServicePath(projectName, serviceName) s3Base := config.ObjectStorageServicePath(projectName, serviceName) + names := append([]string{serviceName}, copies...) - return fsservice.SaveAndUploadFiles(fileTree, serviceName, localBase, s3Base, log.SugaredLogger()) + return fsservice.CopyAndUploadFiles(names, path.Join(localBase, serviceName), s3Base, currentChartPath, log.SugaredLogger()) } func preLoadServiceManifestsFromSource(svc *commonmodels.Service) error { tree, err := fsservice.DownloadFilesFromSource( - &fsservice.DownloadFromSourceArgs{CodehostID: svc.CodehostID, Owner: svc.RepoOwner, Repo: svc.RepoName, Path: svc.LoadPath, Branch: svc.BranchName, RepoLink: svc.SrcPath}, + &fsservice.DownloadFromSourceArgs{CodehostID: svc.CodehostID, Owner: svc.RepoOwner, Namespace: svc.RepoNamespace, Repo: svc.RepoName, Path: svc.LoadPath, Branch: svc.BranchName, RepoLink: svc.SrcPath}, func(afero.Fs) (string, error) { return svc.ServiceName, nil }) @@ -82,10 +131,65 @@ func preLoadServiceManifestsFromSource(svc *commonmodels.Service) error { } // save files to disk and upload them to s3 - if err = SaveAndUploadService(svc.ProductName, svc.ServiceName, tree); err != nil { + if err = SaveAndUploadService(svc.ProductName, svc.ServiceName, nil, tree); err != nil { log.Errorf("Failed to save or upload files for service %s in project %s, error: %s", svc.ServiceName, svc.ProductName, err) return err } return nil } + +func preLoadServiceManifestsFromGerrit(svc *commonmodels.Service) error { + base := path.Join(config.S3StoragePath(), svc.GerritRepoName) + if err := os.RemoveAll(base); err != nil { + log.Warnf("Failed to remove dir, err:%s", err) + } + detail, err := systemconfig.New().GetCodeHost(svc.GerritCodeHostID) + if err != nil { + log.Errorf("Failed to GetCodehostDetail, err:%s", err) + return err + } + err = command.RunGitCmds(detail, "default", "default", svc.GerritRepoName, svc.GerritBranchName, svc.GerritRemoteName) + if err != nil { + log.Errorf("Failed to runGitCmds, err:%s", err) + return err + } + // copy files to disk and upload them to s3 + if err := CopyAndUploadService(svc.ProductName, svc.ServiceName, svc.GerritPath, nil); err != nil { + log.Errorf("Failed to save or upload files for service %s in project %s, error: %s", svc.ServiceName, svc.ProductName, err) + return err + } + return nil +} + +func GeneHelmRepo(chartRepo *commonmodels.HelmRepo) *repo.Entry { + return &repo.Entry{ + Name: chartRepo.RepoName, + URL: chartRepo.URL, + Username: chartRepo.Username, + Password: chartRepo.Password, + } +} + +func preLoadServiceManifestsFromGitee(svc *commonmodels.Service) error { + base := path.Join(config.S3StoragePath(), svc.RepoName) + if err := os.RemoveAll(base); err != nil { + log.Warnf("Failed to remove dir, err:%s", err) + } + detail, err := systemconfig.New().GetCodeHost(svc.CodehostID) + if err != nil { + log.Errorf("Failed to GetCodehostDetail, err:%s", err) + return err + } + err = command.RunGitCmds(detail, svc.RepoOwner, svc.GetRepoNamespace(), svc.RepoName, svc.BranchName, "origin") + if err != nil { + log.Errorf("Failed to runGitCmds, err:%s", err) + return err + } + // save files to disk and upload them to s3 + if err := CopyAndUploadService(svc.ProductName, svc.ServiceName, svc.GiteePath, nil); err != nil { + log.Errorf("Failed to copy or upload files for service %s in project %s, error: %s", svc.ServiceName, svc.ProductName, err) + return err + } + return nil +} diff --git a/pkg/microservice/aslan/core/common/service/image.go b/pkg/microservice/aslan/core/common/service/image.go new file mode 100644 index 0000000000000000000000000000000000000000..9707dafe12cda2976d83dfc10c99bbfbe7e4db0e --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/image.go @@ -0,0 +1,103 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "fmt" + "strings" + + "sigs.k8s.io/yaml" + + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/util/converter" + yamlutil "github.com/koderover/zadig/pkg/util/yaml" +) + +// parse image url to map: repo=>xxx/xx/xx image=>xx tag=>xxx +func resolveImageUrl(imageUrl string) map[string]string { + subMatchAll := imageParseRegex.FindStringSubmatch(imageUrl) + result := make(map[string]string) + exNames := imageParseRegex.SubexpNames() + for i, matchedStr := range subMatchAll { + if i != 0 && matchedStr != "" && matchedStr != ":" { + result[exNames[i]] = matchedStr + } + } + return result +} + +// AssignImageData assign image url data into match data +// matchData: image=>absolute-path repo=>absolute-path tag=>absolute-path +// return: absolute-image-path=>image-value absolute-repo-path=>repo-value absolute-tag-path=>tag-value +func AssignImageData(imageUrl string, matchData map[string]string) (map[string]interface{}, error) { + ret := make(map[string]interface{}) + // total image url assigned into one single value + if len(matchData) == 1 { + for _, v := range matchData { + ret[v] = imageUrl + } + return ret, nil + } + + resolvedImageUrl := resolveImageUrl(imageUrl) + + // image url assigned into repo/image+tag + if len(matchData) == 3 { + ret[matchData[setting.PathSearchComponentRepo]] = strings.TrimSuffix(resolvedImageUrl[setting.PathSearchComponentRepo], "/") + ret[matchData[setting.PathSearchComponentImage]] = resolvedImageUrl[setting.PathSearchComponentImage] + ret[matchData[setting.PathSearchComponentTag]] = resolvedImageUrl[setting.PathSearchComponentTag] + return ret, nil + } + + if len(matchData) == 2 { + // image url assigned into repo/image + tag + if tagPath, ok := matchData[setting.PathSearchComponentTag]; ok { + ret[tagPath] = resolvedImageUrl[setting.PathSearchComponentTag] + for k, imagePath := range matchData { + if k == setting.PathSearchComponentTag { + continue + } + ret[imagePath] = fmt.Sprintf("%s%s", resolvedImageUrl[setting.PathSearchComponentRepo], resolvedImageUrl[setting.PathSearchComponentImage]) + break + } + return ret, nil + } + // image url assigned into repo + image(tag) + ret[matchData[setting.PathSearchComponentRepo]] = strings.TrimSuffix(resolvedImageUrl[setting.PathSearchComponentRepo], "/") + ret[matchData[setting.PathSearchComponentImage]] = fmt.Sprintf("%s:%s", resolvedImageUrl[setting.PathSearchComponentImage], resolvedImageUrl[setting.PathSearchComponentTag]) + return ret, nil + } + + return nil, fmt.Errorf("match data illegal, expect length: 1-3, actual length: %d", len(matchData)) +} + +// ReplaceImage replace image defines in yaml by new version +func ReplaceImage(sourceYaml string, imageValuesMap map[string]interface{}) (string, error) { + nestedMap, err := converter.Expand(imageValuesMap) + if err != nil { + return "", err + } + bs, err := yaml.Marshal(nestedMap) + if err != nil { + return "", err + } + mergedBs, err := yamlutil.Merge([][]byte{[]byte(sourceYaml), bs}) + if err != nil { + return "", err + } + return string(mergedBs), nil +} diff --git a/pkg/microservice/aslan/core/common/service/instantmessage/dingTalk.go b/pkg/microservice/aslan/core/common/service/instantmessage/dingTalk.go new file mode 100644 index 0000000000000000000000000000000000000000..75895de8b3d5244945a2e15920d6a1ff40a47cbf --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/instantmessage/dingTalk.go @@ -0,0 +1,60 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package instantmessage + +const ( + dingDingType = "dingding" +) + +type DingDingMessage struct { + MsgType string `json:"msgtype"` + MarkDown *DingDingMarkDown `json:"markdown"` + At *DingDingAt `json:"at"` +} + +type DingDingMarkDown struct { + Title string `json:"title"` + Text string `json:"text"` +} + +type DingDingAt struct { + AtMobiles []string `json:"atMobiles"` + IsAtAll bool `json:"isAtAll"` +} + +func (w *Service) sendDingDingMessage(uri, title, content string, atMobiles []string) error { + message := &DingDingMessage{ + MsgType: msgType, + MarkDown: &DingDingMarkDown{ + Title: title, + Text: content, + }, + } + if len(atMobiles) > 0 { + message.At = &DingDingAt{ + AtMobiles: atMobiles, + IsAtAll: false, + } + } else { + message.At = &DingDingAt{ + IsAtAll: true, + } + } + + _, err := w.SendMessageRequest(uri, message) + return err +} diff --git a/pkg/microservice/aslan/core/common/service/instantmessage/lark.go b/pkg/microservice/aslan/core/common/service/instantmessage/lark.go new file mode 100644 index 0000000000000000000000000000000000000000..18789800216c05f3c8596c71e3e8a233015c783d --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/instantmessage/lark.go @@ -0,0 +1,214 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package instantmessage + +import ( + "strings" + "sync" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" +) + +const ( + feiShuType = "feishu" + feishuCardType = "interactive" + feishuHeaderTemplateTurquoise = "turquoise" + feishuHeaderTemplateGreen = "green" + feishuHeaderTemplateRed = "red" + feiShuTagText = "plain_text" + feishuTagMd = "lark_md" + feishuTagAction = "action" + feishuTagDiv = "div" + feishuTagButton = "button" +) + +type LarkCardReq struct { + MsgType string `json:"msg_type"` + Card *LarkCard `json:"card"` +} + +type LarkCard struct { + Config *Config `json:"config"` + Header *Header `json:"header"` + I18NElements *I18NElements `json:"i18n_elements"` +} + +type Config struct { + WideScreenMode bool `json:"wide_screen_mode"` +} + +type Title struct { + Content string `json:"content"` + Tag string `json:"tag"` +} + +type Header struct { + Template string `json:"template"` + Title Title `json:"title"` +} + +type TextElem struct { + Content string `json:"content"` + Tag string `json:"tag"` +} + +type Field struct { + IsShort bool `json:"is_short"` + Text TextElem `json:"text"` +} + +type Action struct { + Tag string `json:"tag"` + Text TextElem `json:"text"` + Type string `json:"type"` + URL string `json:"url"` +} + +type ZhCn struct { + Fields []*Field `json:"fields,omitempty"` + Tag string `json:"tag"` + Actions []*Action `json:"actions,omitempty"` +} + +type I18NElements struct { + ZhCn []*ZhCn `json:"zh_cn"` +} + +type FeiShuMessage struct { + Title string `json:"title"` + Text string `json:"text"` +} + +type FeiShuMessageV2 struct { + MsgType string `json:"msg_type"` + Content FeiShuContentV2 `json:"content"` +} + +type FeiShuContentV2 struct { + Text string `json:"text"` +} + +func NewLarkCard() *LarkCard { + return &LarkCard{ + Config: &Config{}, + Header: &Header{}, + I18NElements: &I18NElements{ZhCn: make([]*ZhCn, 0)}, + } +} + +func (lc *LarkCard) SetConfig(wideScreenMode bool) { + if lc.Config == nil { + lc.Config = &Config{} + } + lc.Config.WideScreenMode = wideScreenMode +} + +func (lc *LarkCard) SetHeader(template, title, tag string) { + if lc.Header == nil { + lc.Header = &Header{} + } + lc.Header.Template = template + lc.Header.Title = Title{ + Content: title, + Tag: tag, + } +} + +func (lc *LarkCard) AddI18NElementsZhcnFeild(content string, isCreatefield bool) { + if lc.I18NElements == nil { + lc.I18NElements = &I18NElements{ + ZhCn: make([]*ZhCn, 0), + } + } + + field := &Field{ + IsShort: false, + Text: TextElem{ + Content: content, + Tag: feishuTagMd, + }, + } + var mutex sync.RWMutex + mutex.Lock() + defer mutex.Unlock() + lengthZhCn := len(lc.I18NElements.ZhCn) + if isCreatefield || lengthZhCn == 0 { + zhcnElem := &ZhCn{ + Fields: []*Field{field}, + Tag: feishuTagDiv, + } + lc.I18NElements.ZhCn = append(lc.I18NElements.ZhCn, zhcnElem) + return + } + lengthFields := len(lc.I18NElements.ZhCn[lengthZhCn-1].Fields) + if lengthFields > 0 { + lc.I18NElements.ZhCn[lengthZhCn-1].Fields = append(lc.I18NElements.ZhCn[lengthZhCn-1].Fields, field) + } +} + +func (lc *LarkCard) AddI18NElementsZhcnAction(content, url string) { + if lc.I18NElements == nil { + lc.I18NElements = &I18NElements{ + ZhCn: make([]*ZhCn, 0), + } + } + action := &Action{ + Tag: feishuTagButton, + Text: TextElem{Content: content, Tag: feiShuTagText}, + Type: "primary", + URL: url, + } + zhcnElem := &ZhCn{ + Actions: []*Action{action}, + Tag: feishuTagAction, + } + lc.I18NElements.ZhCn = append(lc.I18NElements.ZhCn, zhcnElem) +} + +func (w *Service) sendFeishuMessage(uri string, lcMsg *LarkCard) error { + message := LarkCardReq{ + MsgType: feishuCardType, + Card: lcMsg, + } + _, err := w.SendMessageRequest(uri, message) + return err +} + +func (w *Service) sendFeishuMessageOfSingleType(title, uri, content string) error { + var message interface{} + message = &FeiShuMessage{ + Title: title, + Text: content, + } + if strings.Contains(uri, "bot/v2/hook") { + message = &FeiShuMessageV2{ + MsgType: "text", + Content: FeiShuContentV2{ + Text: content, + }, + } + } + _, err := w.SendMessageRequest(uri, message) + return err +} + +func getColorTemplateWithStatus(status config.Status) string { + if status == config.StatusPassed { + return feishuHeaderTemplateGreen + } + return feishuHeaderTemplateRed +} diff --git a/pkg/microservice/aslan/core/common/service/instantmessage/service.go b/pkg/microservice/aslan/core/common/service/instantmessage/service.go new file mode 100644 index 0000000000000000000000000000000000000000..5edffb7c4b43bbd26338c8fcedb9072385a4baf0 --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/instantmessage/service.go @@ -0,0 +1,605 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package instantmessage + +import ( + "bytes" + "fmt" + "strconv" + "strings" + "text/template" + "time" + + "k8s.io/apimachinery/pkg/util/sets" + + configbase "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/httpclient" + "github.com/koderover/zadig/pkg/tool/log" +) + +const ( + msgType = "markdown" + singleInfo = "single" + multiInfo = "multi" +) + +type BranchTagType string + +const ( + BranchTagTypeBranch BranchTagType = "Branch" + BranchTagTypeTag BranchTagType = "Tag" + CommitMsgInterceptLength = 60 +) + +type Service struct { + proxyColl *mongodb.ProxyColl + workflowColl *mongodb.WorkflowColl + pipelineColl *mongodb.PipelineColl + testingColl *mongodb.TestingColl + testTaskStatColl *mongodb.TestTaskStatColl +} + +func NewWeChatClient() *Service { + return &Service{ + proxyColl: mongodb.NewProxyColl(), + workflowColl: mongodb.NewWorkflowColl(), + pipelineColl: mongodb.NewPipelineColl(), + testingColl: mongodb.NewTestingColl(), + testTaskStatColl: mongodb.NewTestTaskStatColl(), + } +} + +type wechatNotification struct { + Task *task.Task `json:"task"` + BaseURI string `json:"base_uri"` + IsSingle bool `json:"is_single"` + WebHookType string `json:"web_hook_type"` + TotalTime int64 `json:"total_time"` + AtMobiles []string `json:"atMobiles"` + IsAtAll bool `json:"is_at_all"` +} + +func (w *Service) SendMessageRequest(uri string, message interface{}) ([]byte, error) { + c := httpclient.New() + + // 使用代理 + proxies, _ := w.proxyColl.List(&mongodb.ProxyArgs{}) + if len(proxies) != 0 && proxies[0].EnableApplicationProxy { + c.SetProxy(proxies[0].GetProxyURL()) + fmt.Printf("send message is using proxy:%s\n", proxies[0].GetProxyURL()) + } + + res, err := c.Post(uri, httpclient.SetBody(message)) + if err != nil { + return nil, err + } + + return res.Body(), nil +} + +func (w *Service) SendInstantMessage(task *task.Task, testTaskStatusChanged bool) error { + var notifyCtls []*models.NotifyCtl + var desc string + switch task.Type { + case config.SingleType: + resp, err := w.pipelineColl.Find(&mongodb.PipelineFindOption{Name: task.PipelineName}) + if err != nil { + log.Errorf("failed to find Pipeline,err: %s", err) + return err + } + notifyCtls = resp.NotifyCtls + + case config.WorkflowType: + resp, err := w.workflowColl.Find(task.PipelineName) + if err != nil { + log.Errorf("failed to find Workflow,err: %s", err) + return err + } + notifyCtls = resp.NotifyCtls + + case config.TestType: + resp, err := w.testingColl.Find(strings.TrimSuffix(task.PipelineName, "-job"), task.ProductName) + if err != nil { + log.Errorf("failed to find Testing,err: %s", err) + return err + } + notifyCtls = resp.NotifyCtls + desc = resp.Desc + + default: + log.Errorf("task type is not supported!") + return nil + } + + for _, notifyCtl := range notifyCtls { + if err := w.sendMessage(task, notifyCtl, testTaskStatusChanged, desc); err != nil { + log.Errorf("send %s message err: %s", notifyCtl.WebHookType, err) + continue + } + } + return nil +} + +func (w *Service) sendMessage(task *task.Task, notifyCtl *models.NotifyCtl, testTaskStatusChanged bool, desc string) error { + if notifyCtl == nil { + return nil + } + var ( + uri = "" + content = "" + webHookType = "" + atMobiles []string + isAtAll bool + title = "" + larkCard *LarkCard + err error + ) + if task.Type == config.SingleType { + if notifyCtl.Enabled && sets.NewString(notifyCtl.NotifyTypes...).Has(string(task.Status)) { + webHookType = notifyCtl.WebHookType + if webHookType == dingDingType { + uri = notifyCtl.DingDingWebHook + atMobiles = notifyCtl.AtMobiles + isAtAll = notifyCtl.IsAtAll + } else if webHookType == feiShuType { + uri = notifyCtl.FeiShuWebHook + } else { + uri = notifyCtl.WeChatWebHook + } + content, err = w.createNotifyBody(&wechatNotification{ + Task: task, + BaseURI: configbase.SystemAddress(), + IsSingle: true, + WebHookType: webHookType, + TotalTime: time.Now().Unix() - task.StartTime, + AtMobiles: atMobiles, + IsAtAll: isAtAll, + }) + if err != nil { + log.Errorf("pipeline createNotifyBody err :%s", err) + return err + } + } + } else if task.Type == config.WorkflowType { + if notifyCtl.Enabled && sets.NewString(notifyCtl.NotifyTypes...).Has(string(task.Status)) { + webHookType = notifyCtl.WebHookType + if webHookType == dingDingType { + uri = notifyCtl.DingDingWebHook + atMobiles = notifyCtl.AtMobiles + isAtAll = notifyCtl.IsAtAll + } else if webHookType == feiShuType { + uri = notifyCtl.FeiShuWebHook + } else { + uri = notifyCtl.WeChatWebHook + } + title, content, larkCard, err = w.createNotifyBodyOfWorkflowIM(&wechatNotification{ + Task: task, + BaseURI: configbase.SystemAddress(), + IsSingle: false, + WebHookType: webHookType, + TotalTime: time.Now().Unix() - task.StartTime, + AtMobiles: atMobiles, + IsAtAll: isAtAll, + }) + if err != nil { + log.Errorf("workflow createNotifyBodyOfWorkflowIM err :%s", err) + return err + } + } + } else if task.Type == config.TestType { + statusSets := sets.NewString(notifyCtl.NotifyTypes...) + if notifyCtl.Enabled && (statusSets.Has(string(task.Status)) || (testTaskStatusChanged && statusSets.Has(string(config.StatusChanged)))) { + webHookType = notifyCtl.WebHookType + if webHookType == dingDingType { + uri = notifyCtl.DingDingWebHook + atMobiles = notifyCtl.AtMobiles + isAtAll = notifyCtl.IsAtAll + } else if webHookType == feiShuType { + uri = notifyCtl.FeiShuWebHook + } else { + uri = notifyCtl.WeChatWebHook + } + title, content, larkCard, err = w.createNotifyBodyOfTestIM(desc, &wechatNotification{ + Task: task, + BaseURI: configbase.SystemAddress(), + IsSingle: false, + WebHookType: webHookType, + TotalTime: time.Now().Unix() - task.StartTime, + AtMobiles: atMobiles, + IsAtAll: isAtAll, + }) + if err != nil { + log.Errorf("testing createNotifyBodyOfTestIM err :%s", err) + return err + } + } + } + + if uri != "" && (content != "" || larkCard != nil) { + if webHookType == dingDingType { + if task.Type == config.SingleType { + title = "工作流状态" + } + err := w.sendDingDingMessage(uri, title, content, atMobiles) + if err != nil { + log.Errorf("sendDingDingMessage err : %s", err) + return err + } + } else if webHookType == feiShuType { + if task.Type == config.SingleType { + err := w.sendFeishuMessageOfSingleType("工作流状态", uri, content) + if err != nil { + log.Errorf("sendFeishuMessageOfSingleType Request err : %s", err) + return err + } + return nil + } + + err := w.sendFeishuMessage(uri, larkCard) + if err != nil { + log.Errorf("SendFeiShuMessageRequest err : %s", err) + return err + } + } else { + typeText := weChatTextTypeMarkdown + if task.Type == config.SingleType { + typeText = weChatTextTypeText + } + err := w.SendWeChatWorkMessage(typeText, uri, content) + if err != nil { + log.Errorf("SendWeChatWorkMessage err : %s", err) + return err + } + } + } + return nil +} + +func (w *Service) createNotifyBody(weChatNotification *wechatNotification) (content string, err error) { + tmplSource := "{{if eq .WebHookType \"feishu\"}}触发的工作流: {{.BaseURI}}/v1/projects/detail/{{.Task.ProductName}}/pipelines/{{ isSingle .IsSingle }}/{{.Task.PipelineName}}/{{.Task.TaskID}}{{else}}#### 触发的工作流: [{{.Task.PipelineName}}#{{.Task.TaskID}}]({{.BaseURI}}/v1/projects/detail/{{.Task.ProductName}}/pipelines/{{ isSingle .IsSingle }}/{{.Task.PipelineName}}/{{.Task.TaskID}}){{end}} \n" + + "- 状态: {{if eq .WebHookType \"feishu\"}}{{.Task.Status}}{{else}}{{.Task.Status}}{{end}} \n" + + "- 创建人：{{.Task.TaskCreator}} \n" + + "- 总运行时长：{{ .TotalTime}} 秒 \n" + + testNames := getHTMLTestReport(weChatNotification.Task) + if len(testNames) != 0 { + tmplSource += "- 测试报告：\n" + } + + for _, testName := range testNames { + url := fmt.Sprintf("{{.BaseURI}}/api/aslan/testing/report?pipelineName={{.Task.PipelineName}}&pipelineType={{.Task.Type}}&taskID={{.Task.TaskID}}&testName=%s\n", testName) + if weChatNotification.WebHookType == feiShuType { + tmplSource += url + continue + } + tmplSource += fmt.Sprintf("[%s](%s)\n", url, url) + } + + if weChatNotification.WebHookType == dingDingType { + if len(weChatNotification.AtMobiles) > 0 && !weChatNotification.IsAtAll { + tmplSource = fmt.Sprintf("%s - 相关人员：@%s \n", tmplSource, strings.Join(weChatNotification.AtMobiles, "@")) + } + } + + tplcontent, err := getTplExec(tmplSource, weChatNotification) + return tplcontent, err +} + +func (w *Service) createNotifyBodyOfWorkflowIM(weChatNotification *wechatNotification) (string, string, *LarkCard, error) { + tplTitle := "{{if ne .WebHookType \"feishu\"}}#### {{end}}{{getIcon .Task.Status }}{{if eq .WebHookType \"wechat\"}}工作流{{.Task.PipelineName}} #{{.Task.TaskID}} {{ taskStatus .Task.Status }}{{else}}工作流 {{.Task.PipelineName}} #{{.Task.TaskID}} {{ taskStatus .Task.Status }}{{end}} \n" + tplBaseInfo := []string{"{{if eq .WebHookType \"dingding\"}}##### {{end}}**执行用户**：{{.Task.TaskCreator}} \n", + "{{if eq .WebHookType \"dingding\"}}##### {{end}}**环境信息**：{{.Task.WorkflowArgs.Namespace}} \n", + "{{if eq .WebHookType \"dingding\"}}##### {{end}}**开始时间**：{{ getStartTime .Task.StartTime}} \n", + "{{if eq .WebHookType \"dingding\"}}##### {{end}}**持续时间**：{{ getDuration .TotalTime}} \n", + } + + build := []string{} + test := "" + for _, subStage := range weChatNotification.Task.Stages { + switch subStage.TaskType { + case config.TaskBuild: + for _, sb := range subStage.SubTasks { + buildElemTemp := "" + buildSt, err := base.ToBuildTask(sb) + if err != nil { + return "", "", nil, err + } + branchTag, branchTagType, commitID, commitMsg, gitCommitURL := "", BranchTagTypeBranch, "", "", "" + for idx, buildRepo := range buildSt.JobCtx.Builds { + if idx == 0 || buildRepo.IsPrimary { + branchTag = buildRepo.Branch + if buildRepo.Tag != "" { + branchTagType = BranchTagTypeTag + branchTag = buildRepo.Tag + } + if len(buildRepo.CommitID) > 8 { + commitID = buildRepo.CommitID[0:8] + } + commitMsgs := strings.Split(buildRepo.CommitMessage, "\n") + if len(commitMsgs) > 0 { + commitMsg = commitMsgs[0] + } + if len(commitMsg) > CommitMsgInterceptLength { + commitMsg = commitMsg[0:CommitMsgInterceptLength] + } + gitCommitURL = fmt.Sprintf("%s/%s/%s/commit/%s", buildRepo.Address, buildRepo.RepoOwner, buildRepo.RepoName, commitID) + } + } + if buildSt.BuildStatus.Status == "" { + buildSt.BuildStatus.Status = config.StatusNotRun + } + buildElemTemp += fmt.Sprintf("\n\n{{if eq .WebHookType \"dingding\"}}---\n\n##### {{end}}**服务名称**：%s \n", buildSt.Service) + if !(buildSt.ServiceType == setting.PMDeployType && + (buildSt.JobCtx.FileArchiveCtx != nil || buildSt.JobCtx.DockerBuildCtx != nil)) { + buildElemTemp += fmt.Sprintf("{{if eq .WebHookType \"dingding\"}}##### {{end}}**镜像信息**：%s \n", buildSt.JobCtx.Image) + } + buildElemTemp += fmt.Sprintf("{{if eq .WebHookType \"dingding\"}}##### {{end}}**代码信息**：[%s-%s %s](%s) \n", branchTagType, branchTag, commitID, gitCommitURL) + buildElemTemp += fmt.Sprintf("{{if eq .WebHookType \"dingding\"}}##### {{end}}**提交信息**：%s \n", commitMsg) + build = append(build, buildElemTemp) + } + + case config.TaskJenkinsBuild: + for _, sb := range subStage.SubTasks { + buildElemTemp := "" + buildSt, err := base.ToJenkinsBuildTask(sb) + if err != nil { + return "", "", nil, err + } + buildElemTemp += fmt.Sprintf("\n\n{{if eq .WebHookType \"dingding\"}}---\n\n##### {{end}}**服务名称**：%s \n", buildSt.Service) + buildElemTemp += fmt.Sprintf("{{if eq .WebHookType \"dingding\"}}##### {{end}}**镜像信息**：%s \n", buildSt.Image) + build = append(build, buildElemTemp) + } + + case config.TaskTestingV2: + test = "{{if eq .WebHookType \"dingding\"}}##### {{end}}**测试结果** \n" + for _, sb := range subStage.SubTasks { + test = genTestCaseText(test, sb, weChatNotification.Task.TestReports) + } + } + } + + buttonContent := "点击查看更多信息" + workflowDetailURL := "{{.BaseURI}}/v1/projects/detail/{{.Task.ProductName}}/pipelines/{{ isSingle .IsSingle }}/{{.Task.PipelineName}}/{{.Task.TaskID}}" + moreInformation := fmt.Sprintf("[%s](%s)", buttonContent, workflowDetailURL) + tplTitle, _ = getTplExec(tplTitle, weChatNotification) + + if weChatNotification.WebHookType != feiShuType { + tplcontent := strings.Join(tplBaseInfo, "") + tplcontent += strings.Join(build, "") + tplcontent = fmt.Sprintf("%s%s", tplcontent, test) + if weChatNotification.WebHookType == dingDingType { + if len(weChatNotification.AtMobiles) > 0 && !weChatNotification.IsAtAll { + tplcontent = fmt.Sprintf("%s{{if eq .WebHookType \"dingding\"}}##### {{end}}**相关人员**：@%s \n", tplcontent, strings.Join(weChatNotification.AtMobiles, "@")) + } + } + tplcontent = fmt.Sprintf("%s%s%s", tplTitle, tplcontent, moreInformation) + tplExecContent, _ := getTplExec(tplcontent, weChatNotification) + return tplTitle, tplExecContent, nil, nil + } + + lc := NewLarkCard() + lc.SetConfig(true) + lc.SetHeader(getColorTemplateWithStatus(weChatNotification.Task.Status), tplTitle, feiShuTagText) + for idx, feildContent := range tplBaseInfo { + feildExecContent, _ := getTplExec(feildContent, weChatNotification) + lc.AddI18NElementsZhcnFeild(feildExecContent, idx == 0) + } + for _, feildContent := range build { + feildExecContent, _ := getTplExec(feildContent, weChatNotification) + lc.AddI18NElementsZhcnFeild(feildExecContent, true) + } + if test != "" { + test, _ = getTplExec(test, weChatNotification) + lc.AddI18NElementsZhcnFeild(test, true) + } + workflowDetailURL, _ = getTplExec(workflowDetailURL, weChatNotification) + lc.AddI18NElementsZhcnAction(buttonContent, workflowDetailURL) + return "", "", lc, nil +} + +func (w *Service) createNotifyBodyOfTestIM(desc string, weChatNotification *wechatNotification) (string, string, *LarkCard, error) { + + tplTitle := "{{if ne .WebHookType \"feishu\"}}#### {{end}}{{getIcon .Task.Status }}{{if eq .WebHookType \"wechat\"}}工作流{{.Task.PipelineName}} #{{.Task.TaskID}} {{ taskStatus .Task.Status }}{{else}}工作流 {{.Task.PipelineName}} #{{.Task.TaskID}} {{ taskStatus .Task.Status }}{{end}} \n" + tplBaseInfo := []string{"{{if eq .WebHookType \"dingding\"}}##### {{end}}**执行用户**：{{.Task.TaskCreator}} \n", + "{{if eq .WebHookType \"dingding\"}}##### {{end}}**持续时间**：{{ getDuration .TotalTime}} \n", + "{{if eq .WebHookType \"dingding\"}}##### {{end}}**开始时间**：{{ getStartTime .Task.StartTime}} \n", + "{{if eq .WebHookType \"dingding\"}}##### {{end}}**测试描述**：" + desc + " \n", + } + + tplTestCaseInfo := "{{if eq .WebHookType \"dingding\"}}##### {{end}}**测试结果** \n" + for _, stage := range weChatNotification.Task.Stages { + if stage.TaskType != config.TaskTestingV2 { + continue + } + for _, subTask := range stage.SubTasks { + tplTestCaseInfo = genTestCaseText(tplTestCaseInfo, subTask, weChatNotification.Task.TestReports) + } + } + + buttonContent := "点击查看更多信息" + workflowDetailURL := "{{.BaseURI}}/v1/projects/detail/{{.Task.ProductName}}/test/detail/function/{{.Task.PipelineName}}/{{.Task.TaskID}}" + moreInformation := fmt.Sprintf("{{if eq .WebHookType \"dingding\"}}##### {{end}}[%s](%s)", buttonContent, workflowDetailURL) + + tplTitle, _ = getTplExec(tplTitle, weChatNotification) + + if weChatNotification.WebHookType != feiShuType { + tplcontent := strings.Join(tplBaseInfo, "") + tplcontent = fmt.Sprintf("%s%s", tplcontent, tplTestCaseInfo) + if weChatNotification.WebHookType == dingDingType { + if len(weChatNotification.AtMobiles) > 0 && !weChatNotification.IsAtAll { + tplcontent = fmt.Sprintf("%s{{if eq .WebHookType \"dingding\"}}##### {{end}}**相关人员**：@%s \n", tplcontent, strings.Join(weChatNotification.AtMobiles, "@")) + } + } + tplcontent = fmt.Sprintf("%s%s%s", tplTitle, tplcontent, moreInformation) + tplExecContent, _ := getTplExec(tplcontent, weChatNotification) + return tplTitle, tplExecContent, nil, nil + } + lc := NewLarkCard() + lc.SetConfig(true) + lc.SetHeader(getColorTemplateWithStatus(weChatNotification.Task.Status), tplTitle, feiShuTagText) + for idx, feildContent := range tplBaseInfo { + feildExecContent, _ := getTplExec(feildContent, weChatNotification) + lc.AddI18NElementsZhcnFeild(feildExecContent, idx == 0) + } + if tplTestCaseInfo != "" { + tplTestCaseInfo, _ = getTplExec(tplTestCaseInfo, weChatNotification) + lc.AddI18NElementsZhcnFeild(tplTestCaseInfo, true) + } + workflowDetailURL, _ = getTplExec(workflowDetailURL, weChatNotification) + lc.AddI18NElementsZhcnAction(buttonContent, workflowDetailURL) + + return "", "", lc, nil +} + +func getHTMLTestReport(task *task.Task) []string { + if task.Type != config.WorkflowType { + return nil + } + + testNames := make([]string, 0) + for _, stage := range task.Stages { + if stage.TaskType != config.TaskTestingV2 { + continue + } + + for testName, subTask := range stage.SubTasks { + testInfo, err := base.ToTestingTask(subTask) + if err != nil { + log.Errorf("parse testInfo failed, err:%s", err) + continue + } + + if testInfo.JobCtx.TestType == setting.FunctionTest && testInfo.JobCtx.TestReportPath != "" { + testNames = append(testNames, testName) + } + } + } + + return testNames +} + +func getTplExec(tplcontent string, weChatNotification *wechatNotification) (string, error) { + tmpl := template.Must(template.New("notify").Funcs(template.FuncMap{ + "getColor": func(status config.Status) string { + if status == config.StatusPassed { + return markdownColorInfo + } else if status == config.StatusTimeout || status == config.StatusCancelled { + return markdownColorComment + } else if status == config.StatusFailed { + return markdownColorWarning + } + return markdownColorComment + }, + "isSingle": func(isSingle bool) string { + if isSingle { + return singleInfo + } + return multiInfo + }, + "taskStatus": func(status config.Status) string { + if status == config.StatusPassed { + return "执行成功" + } else if status == config.StatusCancelled { + return "执行取消" + } else if status == config.StatusTimeout { + return "执行超时" + } + return "执行失败" + }, + "getIcon": func(status config.Status) string { + if status == config.StatusPassed { + return "👍" + } + return "⚠️" + }, + "getStartTime": func(startTime int64) string { + return time.Unix(startTime, 0).Format("2006-01-02 15:04:05") + }, + "getDuration": func(startTime int64) string { + duration, er := time.ParseDuration(strconv.FormatInt(startTime, 10) + "s") + if er != nil { + log.Errorf("getTplExec ParseDuration err:%s", er) + return "0s" + } + return duration.String() + }, + }).Parse(tplcontent)) + + buffer := bytes.NewBufferString("") + if err := tmpl.Execute(buffer, &weChatNotification); err != nil { + log.Errorf("getTplExec Execute err:%s", err) + return "", fmt.Errorf("getTplExec Execute err:%s", err) + + } + return buffer.String(), nil +} + +func checkTestReportsExist(testModuleName string, testReports map[string]interface{}) bool { + for testname := range testReports { + if testname == testModuleName { + return true + } + } + return false +} + +func genTestCaseText(test string, subTask, testReports map[string]interface{}) string { + testSt, err := base.ToTestingTask(subTask) + if err != nil { + log.Errorf("parse testInfo failed, err:%s", err) + return test + } + if testSt.TaskStatus == "" { + testSt.TaskStatus = config.StatusNotRun + } + if testSt.JobCtx.TestType == setting.FunctionTest && testSt.JobCtx.TestReportPath != "" && testSt.TaskStatus == config.StatusPassed { + url := fmt.Sprintf("{{.BaseURI}}/api/aslan/testing/report?pipelineName={{.Task.PipelineName}}&pipelineType={{.Task.Type}}&taskID={{.Task.TaskID}}&testName=%s", testSt.TestModuleName) + test += fmt.Sprintf("{{if ne .WebHookType \"feishu\"}} - {{end}}[%s](%s): ", testSt.TestModuleName, url) + } else { + test += fmt.Sprintf("{{if ne .WebHookType \"feishu\"}} - {{end}}%s: ", testSt.TestModuleName) + } + if testReports == nil || !checkTestReportsExist(testSt.TestModuleName, testReports) { + test += fmt.Sprintf("%s \n", testSt.TaskStatus) + return test + } + + for testname, report := range testReports { + if testname != testSt.TestModuleName { + continue + } + tr := &task.TestReport{} + if task.IToi(report, tr) != nil { + log.Errorf("parse TestReport failed, err:%s", err) + continue + } + if tr.FunctionTestSuite == nil { + test += fmt.Sprintf("%s \n", testSt.TaskStatus) + continue + } + totalNum := tr.FunctionTestSuite.Tests + tr.FunctionTestSuite.Skips + failedNum := tr.FunctionTestSuite.Failures + tr.FunctionTestSuite.Errors + successNum := tr.FunctionTestSuite.Tests - failedNum + test += fmt.Sprintf("%d(成功)%d(失败)%d(总数) \n", successNum, failedNum, totalNum) + } + return test +} diff --git a/pkg/microservice/aslan/core/common/service/instantmessage/wechatwork.go b/pkg/microservice/aslan/core/common/service/instantmessage/wechatwork.go new file mode 100644 index 0000000000000000000000000000000000000000..49af4bb1c9003897fee50d379e039b45d71570ab --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/instantmessage/wechatwork.go @@ -0,0 +1,73 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package instantmessage + +import ( + "fmt" +) + +const ( + weChatWorkType = "wechat" + markdownColorInfo = "info" + markdownColorComment = "comment" + markdownColorWarning = "warning" + weChatTextTypeText TextType = "text" + weChatTextTypeMarkdown TextType = "markdown" +) + +type TextType string + +type WeChatWorkCard struct { + MsgType string `json:"msgtype"` + Markdown Markdown `json:"markdown"` +} +type Markdown struct { + Content string `json:"content"` +} + +type Messsage struct { + MsgType string `json:"msgtype"` + Text *Text `json:"markdown"` +} + +type Text struct { + Content string `json:"content"` +} + +func (w *Service) SendWeChatWorkMessage(textType TextType, uri, content string) error { + var message interface{} + if textType == weChatTextTypeText { + message = &Messsage{ + MsgType: msgType, + Text: &Text{ + Content: content, + }, + } + } else if textType == weChatTextTypeMarkdown { + message = &WeChatWorkCard{ + MsgType: msgType, + Markdown: Markdown{ + Content: content, + }, + } + } else { + return fmt.Errorf("SendWeChatWorkMessage err:%s", "WeChatWork textType is invalid") + } + + _, err := w.SendMessageRequest(uri, message) + return err +} diff --git a/pkg/microservice/aslan/core/common/service/kube/actions.go b/pkg/microservice/aslan/core/common/service/kube/actions.go index 51ff6a74119001a7d1f50d84f726ab2f86e7a33d..eeacfa75e5cd628f34375e07ec7be2ddb5b5d3af 100644 --- a/pkg/microservice/aslan/core/common/service/kube/actions.go +++ b/pkg/microservice/aslan/core/common/service/kube/actions.go @@ -17,7 +17,11 @@ limitations under the License. package kube import ( + "context" "fmt" + "regexp" + "strings" + "time" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -27,23 +31,87 @@ import ( "k8s.io/apimachinery/pkg/selection" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/setting" "github.com/koderover/zadig/pkg/tool/kube/getter" "github.com/koderover/zadig/pkg/tool/kube/updater" "github.com/koderover/zadig/pkg/tool/log" + zadigtypes "github.com/koderover/zadig/pkg/types" ) -func CreateNamespace(namespace string, kubeClient client.Client) error { - err := updater.CreateNamespaceByName(namespace, map[string]string{setting.EnvCreatedBy: setting.EnvCreator}, kubeClient) +var registrySecretSuffix = "-registry-secret" + +func CreateNamespace(namespace string, enableShare bool, kubeClient client.Client) error { + labels := map[string]string{ + setting.EnvCreatedBy: setting.EnvCreator, + } + if enableShare { + labels[zadigtypes.IstioLabelKeyInjection] = zadigtypes.IstioLabelValueInjection + } + + err := updater.CreateNamespaceByName(namespace, labels, kubeClient) if err != nil && !apierrors.IsAlreadyExists(err) { return err } + nsObj := &corev1.Namespace{} + // It may fail to obtain the namespace immediately after it is created due to synchronization delay. + // Try twice. + for i := 0; i < 2; i++ { + err = kubeClient.Get(context.TODO(), client.ObjectKey{ + Name: namespace, + }, nsObj) + if err == nil { + break + } + + time.Sleep(time.Second) + } + if err != nil { + return err + } + + if nsObj.Status.Phase == corev1.NamespaceTerminating { + return fmt.Errorf("namespace `%s` is in terminating state, please wait for a whilie and try again.", namespace) + } + return nil } -func CreateOrUpdateRegistrySecret(namespace string, reg *commonmodels.RegistryNamespace, kubeClient client.Client) error { +func CreateOrUpdateRSASecret(publicKey, privateKey []byte, kubeClient client.Client) error { + data := make(map[string][]byte) + + data["publicKey"] = publicKey + data["privateKey"] = privateKey + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: config.Namespace(), + Name: setting.RSASecretName, + }, + Data: data, + Type: corev1.SecretTypeOpaque, + } + return updater.UpdateOrCreateSecret(secret, kubeClient) +} + +func CreateOrUpdateDefaultRegistrySecret(namespace string, reg *commonmodels.RegistryNamespace, kubeClient client.Client) error { + return CreateOrUpdateRegistrySecret(namespace, reg, true, kubeClient) +} + +func CreateOrUpdateRegistrySecret(namespace string, reg *commonmodels.RegistryNamespace, isDefault bool, kubeClient client.Client) error { + var secretName string + var err error + if !isDefault { + secretName, err = genRegistrySecretName(reg) + if err != nil { + return fmt.Errorf("failed to generate registry secret name: %s", err) + } + } else { + secretName = setting.DefaultImagePullSecret + } + data := make(map[string][]byte) dockerConfig := fmt.Sprintf( @@ -58,7 +126,7 @@ func CreateOrUpdateRegistrySecret(namespace string, reg *commonmodels.RegistryNa secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, - Name: setting.DefaultImagePullSecret, + Name: secretName, }, Data: data, Type: corev1.SecretTypeDockercfg, @@ -66,6 +134,48 @@ func CreateOrUpdateRegistrySecret(namespace string, reg *commonmodels.RegistryNa return updater.UpdateOrCreateSecret(secret, kubeClient) } +func genRegistrySecretName(reg *commonmodels.RegistryNamespace) (string, error) { + if reg.IsDefault { + return setting.DefaultImagePullSecret, nil + } + + arr := strings.Split(reg.Namespace, "/") + namespaceInRegistry := arr[len(arr)-1] + + // for AWS ECR, there are no namespace, thus we need to find the NS from the URI + if namespaceInRegistry == "" { + uriDecipher := strings.Split(reg.RegAddr, ".") + namespaceInRegistry = uriDecipher[0] + } + + filteredName, err := formatRegistryName(namespaceInRegistry) + if err != nil { + return "", err + } + + secretName := filteredName + registrySecretSuffix + if reg.RegType != "" { + secretName = filteredName + "-" + reg.RegType + registrySecretSuffix + } + + return secretName, nil +} + +// Note: The name of a Secret object must be a valid DNS subdomain name: +// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names +func formatRegistryName(namespaceInRegistry string) (string, error) { + reg, err := regexp.Compile("[^a-zA-Z0-9\\.-]+") + if err != nil { + return "", err + } + processedName := reg.ReplaceAllString(namespaceInRegistry, "") + processedName = strings.ToLower(processedName) + if len(processedName) > 237 { + processedName = processedName[:237] + } + return processedName, nil +} + // GetDirtyResources searches for dirty active resources in the given namespace, and return their metadata. func GetDirtyResources(ns string, kubeClient client.Client) []metav1.Object { var oms []metav1.Object diff --git a/pkg/microservice/aslan/core/common/service/kube/service.go b/pkg/microservice/aslan/core/common/service/kube/service.go index ee405bfef9085ae53d396a3d771305eb3d8bd1a4..6b5e36f6d897daff5f727049d6a2b51eff286661 100644 --- a/pkg/microservice/aslan/core/common/service/kube/service.go +++ b/pkg/microservice/aslan/core/common/service/kube/service.go @@ -20,6 +20,7 @@ import ( "bytes" "fmt" "net/url" + "strconv" "strings" "text/template" @@ -28,29 +29,30 @@ import ( "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" + config2 "github.com/koderover/zadig/pkg/config" "github.com/koderover/zadig/pkg/microservice/aslan/config" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" "github.com/koderover/zadig/pkg/tool/crypto" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/kube/multicluster" ) -func GetKubeClient(clusterID string) (client.Client, error) { - return multicluster.GetKubeClient(config.HubServerAddress(), clusterID) -} - func GetKubeAPIReader(clusterID string) (client.Reader, error) { - return multicluster.GetKubeAPIReader(config.HubServerAddress(), clusterID) + return kubeclient.GetKubeAPIReader(config.HubServerAddress(), clusterID) } func GetRESTConfig(clusterID string) (*rest.Config, error) { - return multicluster.GetRESTConfig(config.HubServerAddress(), clusterID) + return kubeclient.GetRESTConfig(config.HubServerAddress(), clusterID) } // GetClientset returns a client to interact with APIServer which implements kubernetes.Interface func GetClientset(clusterID string) (kubernetes.Interface, error) { - return multicluster.GetClientset(config.HubServerAddress(), clusterID) + return kubeclient.GetClientset(config.HubServerAddress(), clusterID) } type Service struct { @@ -96,22 +98,40 @@ func (s *Service) ListClusters(clusterType string, logger *zap.SugaredLogger) ([ return clusters, nil } -func (s *Service) CreateCluster(cluster *models.K8SCluster, logger *zap.SugaredLogger) (*models.K8SCluster, error) { +func (s *Service) CreateCluster(cluster *models.K8SCluster, id string, logger *zap.SugaredLogger) (*models.K8SCluster, error) { _, err := s.coll.FindByName(cluster.Name) - if err == nil { logger.Errorf("failed to create cluster %s %v", cluster.Name, err) return nil, e.ErrCreateCluster.AddDesc(e.DuplicateClusterNameFound) } - cluster.Status = config.Pending - - err = s.coll.Create(cluster) - + cluster.Status = setting.Pending + if id == setting.LocalClusterID { + cluster.Status = setting.Normal + cluster.Local = true + cluster.AdvancedConfig = &commonmodels.AdvancedConfig{ + Strategy: "normal", + } + cluster.DindCfg = nil + } + err = s.coll.Create(cluster, id) if err != nil { return nil, e.ErrCreateCluster.AddErr(err) } + if cluster.AdvancedConfig != nil { + for _, projectName := range cluster.AdvancedConfig.ProjectNames { + err = commonrepo.NewProjectClusterRelationColl().Create(&commonmodels.ProjectClusterRelation{ + ProjectName: projectName, + ClusterID: cluster.ID.Hex(), + CreatedBy: cluster.CreatedBy, + }) + if err != nil { + logger.Errorf("Failed to create projectClusterRelation err:%s", err) + } + } + } + token, err := crypto.AesEncrypt(cluster.ID.Hex()) if err != nil { return nil, err @@ -135,7 +155,7 @@ func (s *Service) UpdateCluster(id string, cluster *models.K8SCluster, logger *z return nil, e.ErrUpdateCluster.AddDesc(e.DuplicateClusterNameFound) } - err = s.coll.UpdateMutableFields(cluster) + err = s.coll.UpdateMutableFields(cluster, id) if err != nil { logger.Errorf("failed to update mutable fields %v", err) return nil, e.ErrUpdateCluster.AddErr(err) @@ -174,6 +194,22 @@ func (s *Service) GetCluster(id string, logger *zap.SugaredLogger) (*models.K8SC if err != nil { return nil, err } + + if cluster.DindCfg == nil { + cluster.DindCfg = &commonmodels.DindCfg{ + Replicas: DefaultDindReplicas, + Resources: &commonmodels.Resources{ + Limits: &commonmodels.Limits{ + CPU: DefaultDindLimitsCPU, + Memory: DefaultDindLimitsMemory, + }, + }, + Storage: &commonmodels.DindStorage{ + Type: DefaultDindStorageType, + }, + } + } + cluster.Token = token return cluster, nil } @@ -208,7 +244,7 @@ func (s *Service) ListConnectedClusters(logger *zap.SugaredLogger) ([]*models.K8 return clusters, nil } -func (s *Service) GetYaml(id, agentImage, aslanURL, hubURI string, useDeployment bool, logger *zap.SugaredLogger) ([]byte, error) { +func (s *Service) GetYaml(id, agentImage, rsImage, aslanURL, hubURI string, useDeployment bool, logger *zap.SugaredLogger) ([]byte, error) { var ( cluster *models.K8SCluster err error @@ -235,20 +271,40 @@ func (s *Service) GetYaml(id, agentImage, aslanURL, hubURI string, useDeployment return nil, err } + dindReplicas, dindLimitsCPU, dindLimitsMemory, dindEnablePV, dindSCName, dindStorageSizeInGiB := getDindCfg(cluster) + if cluster.Namespace == "" { err = YamlTemplate.Execute(buffer, TemplateSchema{ - HubAgentImage: agentImage, - ClientToken: token, - HubServerBaseAddr: hubBase.String(), - UseDeployment: useDeployment, + HubAgentImage: agentImage, + ResourceServerImage: rsImage, + ClientToken: token, + HubServerBaseAddr: hubBase.String(), + AslanBaseAddr: config2.SystemAddress(), + UseDeployment: useDeployment, + DindReplicas: dindReplicas, + DindLimitsCPU: dindLimitsCPU, + DindLimitsMemory: dindLimitsMemory, + DindImage: config.DindImage(), + DindEnablePV: dindEnablePV, + DindStorageClassName: dindSCName, + DindStorageSizeInGiB: dindStorageSizeInGiB, }) } else { err = YamlTemplateForNamespace.Execute(buffer, TemplateSchema{ - HubAgentImage: agentImage, - ClientToken: token, - HubServerBaseAddr: hubBase.String(), - UseDeployment: useDeployment, - Namespace: cluster.Namespace, + HubAgentImage: agentImage, + ResourceServerImage: rsImage, + ClientToken: token, + HubServerBaseAddr: hubBase.String(), + AslanBaseAddr: config2.SystemAddress(), + UseDeployment: useDeployment, + Namespace: cluster.Namespace, + DindReplicas: dindReplicas, + DindLimitsCPU: dindLimitsCPU, + DindLimitsMemory: dindLimitsMemory, + DindImage: config.DindImage(), + DindEnablePV: dindEnablePV, + DindStorageClassName: dindSCName, + DindStorageSizeInGiB: dindStorageSizeInGiB, }) } @@ -259,14 +315,74 @@ func (s *Service) GetYaml(id, agentImage, aslanURL, hubURI string, useDeployment return buffer.Bytes(), nil } +func (s *Service) UpdateUpgradeAgentInfo(id, updateHubagentErrorMsg string) error { + _, err := s.coll.Get(id) + if err != nil { + return err + } + err = s.coll.UpdateUpgradeAgentInfo(id, updateHubagentErrorMsg) + return err +} + +func getDindCfg(cluster *models.K8SCluster) (replicas int, limitsCPU, limitsMemory string, enablePV bool, scName string, storageSizeInGiB int) { + replicas = DefaultDindReplicas + limitsCPU = strconv.Itoa(DefaultDindLimitsCPU) + setting.CpuUintM + limitsMemory = strconv.Itoa(DefaultDindLimitsMemory) + setting.MemoryUintMi + enablePV = DefaultDindEnablePV + scName = DefaultDindStorageClassName + storageSizeInGiB = DefaultDindStorageSizeInGiB + + if cluster.DindCfg != nil { + if cluster.DindCfg.Replicas > 0 { + replicas = cluster.DindCfg.Replicas + } + + if cluster.DindCfg.Resources != nil && cluster.DindCfg.Resources.Limits != nil { + if cluster.DindCfg.Resources.Limits.CPU > 0 { + limitsCPU = strconv.Itoa(cluster.DindCfg.Resources.Limits.CPU) + setting.CpuUintM + } + if cluster.DindCfg.Resources.Limits.Memory > 0 { + limitsMemory = strconv.Itoa(cluster.DindCfg.Resources.Limits.Memory) + setting.MemoryUintMi + } + } + + if cluster.DindCfg.Storage != nil && cluster.DindCfg.Storage.Type == commonmodels.DindStorageDynamic { + enablePV = true + scName = cluster.DindCfg.Storage.StorageClass + storageSizeInGiB = int(cluster.DindCfg.Storage.StorageSizeInGiB) + } + } + + return +} + type TemplateSchema struct { - HubAgentImage string - ClientToken string - HubServerBaseAddr string - Namespace string - UseDeployment bool + HubAgentImage string + ResourceServerImage string + ClientToken string + HubServerBaseAddr string + Namespace string + UseDeployment bool + AslanBaseAddr string + DindReplicas int + DindLimitsCPU string + DindLimitsMemory string + DindImage string + DindEnablePV bool + DindStorageClassName string + DindStorageSizeInGiB int } +const ( + DefaultDindReplicas int = 1 + DefaultDindLimitsCPU int = 4000 + DefaultDindLimitsMemory int = 8192 + DefaultDindStorageType commonmodels.DindStorageType = commonmodels.DindStorageRootfs + DefaultDindEnablePV bool = false + DefaultDindStorageClassName string = "" + DefaultDindStorageSizeInGiB int = 10 +) + var YamlTemplate = template.Must(template.New("agentYaml").Parse(` --- @@ -285,7 +401,7 @@ metadata: --- -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: koderover-agent-admin-binding @@ -318,6 +434,25 @@ rules: - '*' --- + +apiVersion: v1 +kind: Service +metadata: + name: hub-agent + namespace: koderover-agent + labels: + app: koderover-agent-agent +spec: + type: ClusterIP + ports: + - protocol: TCP + port: 80 + targetPort: 80 + selector: + app: koderover-agent-agent + +--- + apiVersion: apps/v1 {{- if .UseDeployment }} kind: Deployment @@ -367,19 +502,158 @@ spec: value: "{{.ClientToken}}" - name: HUB_SERVER_BASE_ADDR value: "{{.HubServerBaseAddr}}" + - name: ASLAN_BASE_ADDR + value: "{{.AslanBaseAddr}}" resources: limits: cpu: 1000m memory: 1Gi requests: cpu: 100m - memory: 256Mi + memory: 256Mi {{- if .UseDeployment }} replicas: 1 {{- else }} updateStrategy: type: RollingUpdate {{- end }} + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: resource-server + namespace: koderover-agent + labels: + app.kubernetes.io/component: resource-server + app.kubernetes.io/name: zadig +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: resource-server + app.kubernetes.io/name: zadig + template: + metadata: + labels: + app.kubernetes.io/component: resource-server + app.kubernetes.io/name: zadig + spec: + containers: + - image: {{.ResourceServerImage}} + imagePullPolicy: Always + name: resource-server + resources: + limits: + cpu: 500m + memory: 500Mi + requests: + cpu: 100m + memory: 100Mi + +--- + +apiVersion: v1 +kind: Service +metadata: + name: resource-server + namespace: koderover-agent + labels: + app.kubernetes.io/component: resource-server + app.kubernetes.io/name: zadig +spec: + type: ClusterIP + ports: + - protocol: TCP + port: 80 + targetPort: 80 + selector: + app.kubernetes.io/component: resource-server + app.kubernetes.io/name: zadig + +--- + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: dind + namespace: koderover-agent + labels: + app.kubernetes.io/component: dind + app.kubernetes.io/name: zadig +spec: + serviceName: dind + replicas: {{.DindReplicas}} + selector: + matchLabels: + app.kubernetes.io/component: dind + app.kubernetes.io/name: zadig + template: + metadata: + labels: + app.kubernetes.io/component: dind + app.kubernetes.io/name: zadig + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + containers: + - name: dind + image: {{.DindImage}} + env: + - name: DOCKER_TLS_CERTDIR + value: "" + securityContext: + privileged: true + ports: + - protocol: TCP + containerPort: 2375 + resources: + limits: + cpu: {{.DindLimitsCPU}} + memory: {{.DindLimitsMemory}} + requests: + cpu: 100m + memory: 128Mi +{{- if .DindEnablePV }} + volumeMounts: + - name: zadig-docker + mountPath: /var/lib/docker + volumeClaimTemplates: + - metadata: + name: zadig-docker + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: {{.DindStorageClassName}} + resources: + requests: + storage: {{.DindStorageSizeInGiB}}Gi +{{- end }} + +--- + +apiVersion: v1 +kind: Service +metadata: + name: dind + namespace: koderover-agent + labels: + app.kubernetes.io/component: dind + app.kubernetes.io/name: zadig +spec: + ports: + - name: dind + protocol: TCP + port: 2375 + targetPort: 2375 + clusterIP: None + selector: + app.kubernetes.io/component: dind + app.kubernetes.io/name: zadig `)) var YamlTemplateForNamespace = template.Must(template.New("agentYaml").Parse(` @@ -393,7 +667,7 @@ metadata: --- -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: koderover-agent-admin-binding @@ -423,6 +697,25 @@ rules: - '*' --- + +apiVersion: v1 +kind: Service +metadata: + name: hub-agent + namespace: {{.Namespace}} + labels: + app: koderover-agent-agent +spec: + type: ClusterIP + ports: + - protocol: TCP + port: 80 + targetPort: 80 + selector: + app: koderover-agent-agent + +--- + apiVersion: apps/v1 {{- if .UseDeployment }} kind: Deployment @@ -472,17 +765,157 @@ spec: value: "{{.ClientToken}}" - name: HUB_SERVER_BASE_ADDR value: "{{.HubServerBaseAddr}}" + - name: ASLAN_BASE_ADDR + value: "{{.AslanBaseAddr}}" resources: limits: cpu: 1000m memory: 1Gi requests: cpu: 100m - memory: 256Mi + memory: 256Mi {{- if .UseDeployment }} replicas: 1 {{- else }} updateStrategy: type: RollingUpdate {{- end }} + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: resource-server + namespace: {{.Namespace}} + labels: + app.kubernetes.io/component: resource-server + app.kubernetes.io/name: zadig +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: resource-server + app.kubernetes.io/name: zadig + template: + metadata: + labels: + app.kubernetes.io/component: resource-server + app.kubernetes.io/name: zadig + spec: + containers: + - image: {{.ResourceServerImage}} + imagePullPolicy: Always + name: resource-server + resources: + limits: + cpu: 500m + memory: 500Mi + requests: + cpu: 100m + memory: 100Mi + +--- + +apiVersion: v1 +kind: Service +metadata: + name: resource-server + namespace: {{.Namespace}} + labels: + app.kubernetes.io/component: resource-server + app.kubernetes.io/instance: zadig-zadig + app.kubernetes.io/name: zadig +spec: + type: ClusterIP + ports: + - protocol: TCP + port: 80 + targetPort: 80 + selector: + app.kubernetes.io/component: resource-server + app.kubernetes.io/name: zadig + +--- + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: dind + namespace: {{.Namespace}} + labels: + app.kubernetes.io/component: dind + app.kubernetes.io/name: zadig +spec: + serviceName: dind + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: dind + app.kubernetes.io/name: zadig + template: + metadata: + labels: + app.kubernetes.io/component: dind + app.kubernetes.io/name: zadig + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + containers: + - name: dind + image: {{.DindImage}} + env: + - name: DOCKER_TLS_CERTDIR + value: "" + securityContext: + privileged: true + ports: + - protocol: TCP + containerPort: 2375 + resources: + limits: + cpu: "4" + memory: 8Gi + requests: + cpu: 100m + memory: 128Mi +{{- if .DindEnablePV }} + volumeMounts: + - name: zadig-docker + mountPath: /var/lib/docker + volumeClaimTemplates: + - metadata: + name: zadig-docker + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: {{.DindStorageClassName}} + resources: + requests: + storage: {{.DindStorageSizeInGiB}}Gi +{{- end }} + +--- + +apiVersion: v1 +kind: Service +metadata: + name: dind + namespace: {{.Namespace}} + labels: + app.kubernetes.io/component: dind + app.kubernetes.io/name: zadig +spec: + ports: + - name: dind + protocol: TCP + port: 2375 + targetPort: 2375 + clusterIP: None + selector: + app.kubernetes.io/component: dind + app.kubernetes.io/name: zadig `)) diff --git a/pkg/microservice/aslan/core/common/service/kube/status.go b/pkg/microservice/aslan/core/common/service/kube/status.go index cf2460ad5b81e4570a32ef40eb53aae71e849ec0..58c17a4230c877661924512a73a2907a4941f463 100644 --- a/pkg/microservice/aslan/core/common/service/kube/status.go +++ b/pkg/microservice/aslan/core/common/service/kube/status.go @@ -21,16 +21,15 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" - "sigs.k8s.io/controller-runtime/pkg/client" + "k8s.io/client-go/informers" "github.com/koderover/zadig/pkg/setting" "github.com/koderover/zadig/pkg/shared/kube/wrapper" "github.com/koderover/zadig/pkg/tool/kube/getter" ) -// TODO: improve this function -func GetSelectedPodsInfo(ns string, selector labels.Selector, kubeClient client.Client, log *zap.SugaredLogger) (string, string, []string) { - pods, err := getter.ListPods(ns, selector, kubeClient) +func GetSelectedPodsInfo(selector labels.Selector, informer informers.SharedInformerFactory, log *zap.SugaredLogger) (string, string, []string) { + pods, err := getter.ListPodsWithCache(selector, informer) if err != nil { return setting.PodError, setting.PodNotReady, nil } @@ -52,19 +51,16 @@ func GetSelectedPodsInfo(ns string, selector labels.Selector, kubeClient client. for _, pod := range pods { iPod := wrapper.Pod(pod) - // skip if pod is a succeeded job if iPod.Succeeded() { succeededPods++ continue } - // 如果服务中任意一个pod不在running状态，返回unstable if !iPod.Ready() { return setting.PodUnstable, setting.PodNotReady, images } } - // return "Succeeded, Completed" if all pods are succeeded Job pods if len(pods) == succeededPods { return string(corev1.PodSucceeded), setting.JobReady, images } diff --git a/pkg/microservice/aslan/core/common/service/notify/client.go b/pkg/microservice/aslan/core/common/service/notify/client.go index d08af8522955436ea4009e362637abaca257285d..0f40a2d947b891fe86a4ba6cfd3c5400bd0124f1 100644 --- a/pkg/microservice/aslan/core/common/service/notify/client.go +++ b/pkg/microservice/aslan/core/common/service/notify/client.go @@ -17,36 +17,50 @@ limitations under the License. package notify import ( + "bytes" "encoding/json" "fmt" + "net/http" + "net/url" + + "github.com/pkg/errors" "github.com/koderover/zadig/pkg/microservice/aslan/config" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/instantmessage" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/scmnotify" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/wechat" + "github.com/koderover/zadig/pkg/setting" "github.com/koderover/zadig/pkg/tool/log" ) const sevendays int64 = 60 * 60 * 24 * 7 +type WorkflowTaskImage struct { + Image string `json:"image"` + ServiceName string `json:"service_name"` + RegistryRepo string `json:"registry_repo"` +} + type client struct { - notifyColl *mongodb.NotifyColl - pipelineColl *mongodb.PipelineColl - subscriptionColl *mongodb.SubscriptionColl - taskColl *mongodb.TaskColl - scmNotifyService *scmnotify.Service - WeChatService *wechat.Service + notifyColl *mongodb.NotifyColl + pipelineColl *mongodb.PipelineColl + subscriptionColl *mongodb.SubscriptionColl + taskColl *mongodb.TaskColl + scmNotifyService *scmnotify.Service + InstantmessageService *instantmessage.Service } func NewNotifyClient() *client { return &client{ - notifyColl: mongodb.NewNotifyColl(), - pipelineColl: mongodb.NewPipelineColl(), - subscriptionColl: mongodb.NewSubscriptionColl(), - taskColl: mongodb.NewTaskColl(), - scmNotifyService: scmnotify.NewService(), - WeChatService: wechat.NewWeChatClient(), + notifyColl: mongodb.NewNotifyColl(), + pipelineColl: mongodb.NewPipelineColl(), + subscriptionColl: mongodb.NewSubscriptionColl(), + taskColl: mongodb.NewTaskColl(), + scmNotifyService: scmnotify.NewService(), + InstantmessageService: instantmessage.NewWeChatClient(), } } @@ -183,6 +197,7 @@ func (c *client) ProccessNotify(notify *models.Notify) error { receivers := []string{notify.Receiver} logger := log.SugaredLogger() task.Status = ctx.Status + testTaskStatusChanged := false if ctx.Type == config.SingleType { pipline, err := c.pipelineColl.Find(&mongodb.PipelineFindOption{Name: ctx.PipelineName}) if err != nil { @@ -194,18 +209,52 @@ func (c *client) ProccessNotify(notify *models.Notify) error { logger.Infof("pipeline get task #%d notify, status: %s", ctx.TaskID, ctx.Status) _ = c.scmNotifyService.UpdatePipelineWebhookComment(task, logger) } else if ctx.Type == config.WorkflowType { + if task.TaskCreator == setting.RequestModeOpenAPI { + // send callback requests + err = c.sendCallbackRequest(task) + if err != nil { + logger.Errorf("failed to send callback request for workflow: %s, taskID: %d, err: %s", task.PipelineName, task.TaskID, err) + } + return nil + } logger.Infof("workflow get task #%d notify, status: %s", ctx.TaskID, ctx.Status) _ = c.scmNotifyService.UpdateWebhookComment(task, logger) - _ = c.scmNotifyService.UpdateDiffNote(task, logger) + task.Stages = ctx.Stages } else if ctx.Type == config.TestType { logger.Infof("test get task #%d notify, status: %s", ctx.TaskID, ctx.Status) _ = c.scmNotifyService.UpdateWebhookCommentForTest(task, logger) + if ctx.TaskID > 1 { + testPreTask, err := c.taskColl.Find(ctx.TaskID-1, ctx.PipelineName, ctx.Type) + if err != nil { + return fmt.Errorf("get test previous task #%d notify, status: %s,err:%s", ctx.TaskID-1, ctx.Status, err) + } + if testPreTask.Status != task.Status && task.Status != config.StatusRunning { + testTaskStatusChanged = true + } + } + task.Stages = ctx.Stages + } else if ctx.Type == config.ScanningType { + logger.Infof("scanning get task #%d notify, status: %s", ctx.TaskID, ctx.Status) + err = c.scmNotifyService.UpdateWebhookCommentForScanning(task, logger) + if err != nil { + // FIXME: This error will not be returned since the logic above didn't return. It is expected to return if we know what we are doing. + log.Errorf("Failed to update webhook comment for scanning: %s, err: %s", task.ScanningArgs.ScanningID, err) + } + if ctx.TaskID > 1 { + scanningPreTask, err := c.taskColl.Find(ctx.TaskID-1, ctx.PipelineName, ctx.Type) + if err != nil { + return fmt.Errorf("get test previous task #%d notify, status: %s,err:%s", ctx.TaskID-1, ctx.Status, err) + } + if scanningPreTask.Status != task.Status && task.Status != config.StatusRunning { + testTaskStatusChanged = true + } + } + task.Stages = ctx.Stages } - //发送微信通知 - err = c.WeChatService.SendWechatMessage(task) + err = c.InstantmessageService.SendInstantMessage(task, testTaskStatusChanged) if err != nil { - return fmt.Errorf("SendWechatMessage err : %v", err) + return fmt.Errorf("SendInstantMessage err : %s", err) } for _, receiver := range receivers { @@ -262,3 +311,97 @@ func (c *client) sendSubscribedNotify(notify *models.Notify) error { } return nil } + +func taskFinished(task *task.Task) bool { + status := task.Status + if status == config.StatusPassed || status == config.StatusFailed || status == config.StatusTimeout || status == config.StatusCancelled { + return true + } + return false +} + +func getImages(task *task.Task) ([]*WorkflowTaskImage, error) { + ret := make([]*WorkflowTaskImage, 0) + for _, stages := range task.Stages { + if stages.TaskType != config.TaskBuild || stages.Status != config.StatusPassed { + continue + } + for _, subTask := range stages.SubTasks { + buildInfo, err := base.ToBuildTask(subTask) + if err != nil { + log.Errorf("get buildInfo ToBuildTask failed ! err:%s", err) + return nil, err + } + + WorkflowTaskImage := &WorkflowTaskImage{ + Image: buildInfo.JobCtx.Image, + ServiceName: buildInfo.ServiceName, + } + if buildInfo.DockerBuildStatus != nil { + WorkflowTaskImage.RegistryRepo = buildInfo.DockerBuildStatus.RegistryRepo + } + ret = append(ret, WorkflowTaskImage) + } + } + return ret, nil +} + +// send callback request when workflow is finished +func (c *client) sendCallbackRequest(task *task.Task) error { + if !taskFinished(task) { + return nil + } + + //TODO remove this log after openApi becomes stable + log.Infof("sending callback request for workflow:%s, taskID:%d", task.PipelineName, task.TaskID) + + if task.WorkflowArgs == nil { + return nil + } + + callback := task.WorkflowArgs.Callback + if callback == nil { + return nil + } + + callbackUrl, err := url.PathUnescape(callback.CallbackUrl) + if err != nil { + return errors.Wrapf(err, "failed to unescape callback url") + } + + responseBody := make(map[string]interface{}) + + // set task status data + responseBody["status"] = task.Status + responseBody["task_id"] = task.TaskID + responseBody["workflow_name"] = task.PipelineName + responseBody["error"] = task.Error + + // set custom kvs + responseBody["vars"] = callback.CallbackVars + + // set images + responseBody["images"], err = getImages(task) + + reqBody, err := json.Marshal(responseBody) + if err != nil { + log.Errorf("marshal json args error: %s", err) + return err + } + + req, err := http.NewRequest("POST", callbackUrl, bytes.NewBuffer(reqBody)) + req.Header.Set("Content-Type", "application/json") + + var resp *http.Response + client := &http.Client{} + resp, err = client.Do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("callback response code error: %d", resp.StatusCode) + } + return nil +} diff --git a/pkg/microservice/aslan/core/common/service/pipeline.go b/pkg/microservice/aslan/core/common/service/pipeline.go index 3fc47b6c11949229cc589f39f8e359c10ad3498e..dd7b9e1549af95530f9730980873bf35963d9dc0 100644 --- a/pkg/microservice/aslan/core/common/service/pipeline.go +++ b/pkg/microservice/aslan/core/common/service/pipeline.go @@ -111,7 +111,7 @@ func DeletePipeline(pipelineName, requestID string, isDeletingProductTmpl bool, return nil } -func GetPipelineInfo(userID int, pipelineName string, log *zap.SugaredLogger) (*commonmodels.Pipeline, error) { +func GetPipelineInfo(pipelineName string, log *zap.SugaredLogger) (*commonmodels.Pipeline, error) { resp, err := mongodb.NewPipelineColl().Find(&mongodb.PipelineFindOption{Name: pipelineName}) if err != nil { log.Error(err) diff --git a/pkg/microservice/aslan/core/common/service/pm/env_status.go b/pkg/microservice/aslan/core/common/service/pm/env_status.go new file mode 100644 index 0000000000000000000000000000000000000000..934e1758ce00f8886549e6fd3d1502a3da593c56 --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/pm/env_status.go @@ -0,0 +1,56 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pm + +import ( + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/util/sets" + + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" +) + +func GenerateEnvStatus(envConfigs []*commonmodels.EnvConfig, log *zap.SugaredLogger) ([]*commonmodels.EnvStatus, error) { + changeEnvStatus := []*commonmodels.EnvStatus{} + for _, envConfig := range envConfigs { + tmpPrivateKeys, err := commonrepo.NewPrivateKeyColl().ListHostIPByArgs(&commonrepo.ListHostIPArgs{IDs: envConfig.HostIDs}) + if err != nil { + log.Errorf("ListNameByArgs ids err:%s", err) + return nil, err + } + + privateKeysByLabels, err := commonrepo.NewPrivateKeyColl().ListHostIPByArgs(&commonrepo.ListHostIPArgs{Labels: envConfig.Labels}) + if err != nil { + log.Errorf("ListNameByArgs labels err:%s", err) + return nil, err + } + tmpPrivateKeys = append(tmpPrivateKeys, privateKeysByLabels...) + privateKeysSet := sets.NewString() + for _, v := range tmpPrivateKeys { + tmp := commonmodels.EnvStatus{ + HostID: v.ID.Hex(), + EnvName: envConfig.EnvName, + Address: v.IP, + } + if !privateKeysSet.Has(tmp.HostID) { + changeEnvStatus = append(changeEnvStatus, &tmp) + privateKeysSet.Insert(tmp.HostID) + } + } + } + return changeEnvStatus, nil +} diff --git a/pkg/microservice/aslan/core/common/service/product.go b/pkg/microservice/aslan/core/common/service/product.go index c80b56372db4c23b9df4af1a44ecb812befcff2d..b1b1c724ebf7ef2edc3c24b54b7a205fae78efc9 100644 --- a/pkg/microservice/aslan/core/common/service/product.go +++ b/pkg/microservice/aslan/core/common/service/product.go @@ -17,209 +17,29 @@ limitations under the License. package service import ( + "context" "fmt" - "time" + "strings" "github.com/hashicorp/go-multierror" - helmclient "github.com/mittwald/go-helm-client" "go.uber.org/zap" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - "sigs.k8s.io/controller-runtime/pkg/client" + "k8s.io/apimachinery/pkg/util/sets" "github.com/koderover/zadig/pkg/microservice/aslan/config" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/poetry" - e "github.com/koderover/zadig/pkg/tool/errors" - helmtool "github.com/koderover/zadig/pkg/tool/helmclient" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" "github.com/koderover/zadig/pkg/tool/kube/updater" + "github.com/koderover/zadig/pkg/util" ) -const ( - timeout = 60 -) - -func DeleteProduct(username, envName, productName, requestID string, log *zap.SugaredLogger) (err error) { - eventStart := time.Now().Unix() - productInfo, err := mongodb.NewProductColl().Find(&mongodb.ProductFindOptions{Name: productName, EnvName: envName}) - if err != nil { - log.Errorf("find product error: %v", err) - return e.ErrDeleteEnv.AddDesc("not found") - } - - kubeClient, err := kube.GetKubeClient(productInfo.ClusterID) - if err != nil { - return e.ErrDeleteEnv.AddErr(err) - } - - restConfig, err := kube.GetRESTConfig(productInfo.ClusterID) - if err != nil { - return e.ErrDeleteEnv.AddErr(err) - } - - // 设置产品状态 - err = mongodb.NewProductColl().UpdateStatus(envName, productName, setting.ProductStatusDeleting) - if err != nil { - log.Errorf("[%s][%s] update product status error: %v", username, productInfo.Namespace, err) - return e.ErrDeleteEnv.AddDesc("更新集成环境状态失败: " + err.Error()) - } - - log.Infof("[%s] delete product %s", username, productInfo.Namespace) - LogProductStats(username, setting.DeleteProductEvent, productName, requestID, eventStart, log) - - poetryClient := poetry.New(config.PoetryAPIServer(), config.PoetryAPIRootKey()) - - switch productInfo.Source { - case setting.SourceFromHelm: - err = mongodb.NewProductColl().Delete(envName, productName) - if err != nil { - log.Errorf("Product.Delete error: %v", err) - } - - _, err = poetryClient.DeleteEnvRolePermission(productName, envName, log) - if err != nil { - log.Errorf("DeleteEnvRole error: %v", err) - } - - go func() { - var err error - defer func() { - if err != nil { - // 发送删除产品失败消息给用户 - title := fmt.Sprintf("删除项目:[%s] 环境:[%s] 失败!", productName, envName) - SendErrorMessage(username, title, requestID, err, log) - _ = mongodb.NewProductColl().UpdateStatus(envName, productName, setting.ProductStatusUnknown) - } else { - title := fmt.Sprintf("删除项目:[%s] 环境:[%s] 成功!", productName, envName) - content := fmt.Sprintf("namespace:%s", productInfo.Namespace) - SendMessage(username, title, content, requestID, log) - } - }() - - //卸载helm release资源 - if hc, err := helmtool.NewClientFromRestConf(restConfig, productInfo.Namespace); err == nil { - for _, services := range productInfo.Services { - for _, service := range services { - if err = hc.UninstallRelease(&helmclient.ChartSpec{ - ReleaseName: fmt.Sprintf("%s-%s", productInfo.Namespace, service.ServiceName), - Namespace: productInfo.Namespace, - Wait: true, - Force: true, - Timeout: timeout * time.Second * 10, - }); err != nil { - log.Errorf("UninstallRelease err:%v", err) - } - } - } - } else { - log.Errorf("获取helmClient err:%v", err) - } - - //删除namespace - s := labels.Set{setting.EnvCreatedBy: setting.EnvCreator}.AsSelector() - if err1 := updater.DeleteMatchingNamespace(productInfo.Namespace, s, kubeClient); err1 != nil { - err = e.ErrDeleteEnv.AddDesc(e.DeleteNamespaceErrMsg + ": " + err1.Error()) - return - } - }() - case setting.SourceFromExternal: - err = mongodb.NewProductColl().Delete(envName, productName) - if err != nil { - log.Errorf("Product.Delete error: %v", err) - } - - _, err = poetryClient.DeleteEnvRolePermission(productName, envName, log) - if err != nil { - log.Errorf("DeleteEnvRole error: %v", err) - } - - // 删除workload数据 - tempProduct, err := template.NewProductColl().Find(productName) - if err != nil { - log.Errorf("project not found error:%s", err) - } - if tempProduct.ProductFeature != nil && tempProduct.ProductFeature.CreateEnvType == setting.SourceFromExternal { - workloadStat, err := mongodb.NewWorkLoadsStatColl().Find(productInfo.ClusterID, productInfo.Namespace) - if err != nil { - log.Errorf("workflowStat not found error:%s", err) - } - if workloadStat != nil { - workloadStat.Workloads = filterWorkloadsByEnv(workloadStat.Workloads, productInfo.EnvName) - if err := mongodb.NewWorkLoadsStatColl().UpdateWorkloads(workloadStat); err != nil { - log.Errorf("update workloads fail error:%s", err) - } - } - // 删除所有external的服务 - err = commonrepo.NewServiceColl().UpdateExternalServicesStatus("", productName, setting.ProductStatusDeleting, envName) - if err != nil { - log.Errorf("UpdateStatus external services error:%s", err) - } - } - - default: - go func() { - var err error - defer func() { - if err != nil { - // 发送删除产品失败消息给用户 - title := fmt.Sprintf("删除项目:[%s] 环境:[%s] 失败!", productName, envName) - SendErrorMessage(username, title, requestID, err, log) - _ = mongodb.NewProductColl().UpdateStatus(envName, productName, setting.ProductStatusUnknown) - } else { - title := fmt.Sprintf("删除项目:[%s] 环境:[%s] 成功!", productName, envName) - content := fmt.Sprintf("namespace:%s", productInfo.Namespace) - SendMessage(username, title, content, requestID, log) - } - }() - - err = DeleteResourcesAsync(productInfo.Namespace, labels.Set{setting.ProductLabel: productName}.AsSelector(), kubeClient, log) - if err != nil { - err = e.ErrDeleteProduct.AddDesc(e.DeleteServiceContainerErrMsg + ": " + err.Error()) - return - } - - err = DeleteClusterResourceAsync(labels.Set{setting.ProductLabel: productName, setting.EnvNameLabel: envName}.AsSelector(), kubeClient, log) - - if err != nil { - err = e.ErrDeleteProduct.AddDesc(e.DeleteServiceContainerErrMsg + ": " + err.Error()) - return - } +const SplitSymbol = "&" - s := labels.Set{setting.EnvCreatedBy: setting.EnvCreator}.AsSelector() - if err1 := updater.DeleteMatchingNamespace(productInfo.Namespace, s, kubeClient); err1 != nil { - err = e.ErrDeleteEnv.AddDesc(e.DeleteNamespaceErrMsg + ": " + err1.Error()) - return - } - - //// 等待10分钟 - //err = WaitPodsToTerminate(productInfo.Namespace, selector, config.ServiceStartTimeout(), kubeClient, log) - //if err != nil { - // log.Errorf("wait port to terminate error: %v", err) - // err = e.ErrDeleteEnv.AddDesc(e.DeleteServiceContainerErrMsg + ": " + err.Error()) - // return - //} - - err = mongodb.NewProductColl().Delete(envName, productName) - if err != nil { - log.Errorf("Product.Delete error: %v", err) - } - - _, err = poetryClient.DeleteEnvRolePermission(productName, envName, log) - if err != nil { - log.Errorf("DeleteEnvRole error: %v", err) - } - }() - } - return nil -} - -func filterWorkloadsByEnv(exist []models.Workload, env string) []models.Workload { - result := make([]models.Workload, 0) +func FilterWorkloadsByEnv(exist []commonmodels.Workload, env string) []commonmodels.Workload { + result := make([]commonmodels.Workload, 0) for _, v := range exist { if v.EnvName != env { result = append(result, v) @@ -228,96 +48,104 @@ func filterWorkloadsByEnv(exist []models.Workload, env string) []models.Workload return result } -func DeleteClusterResourceAsync(selector labels.Selector, kubeClient client.Client, log *zap.SugaredLogger) error { - log.Infof("[%s] delete cluster kube resources", selector) +func DeleteClusterResource(selector labels.Selector, clusterID string, log *zap.SugaredLogger) error { + log.Infof("Deleting cluster resources with selector: [%s]", selector) + + clientset, err := kubeclient.GetKubeClientSet(config.HubServerAddress(), clusterID) + if err != nil { + log.Errorf("failed to create kubernetes clientset for clusterID: %s, the error is: %s", clusterID, err) + return err + } errors := new(multierror.Error) - if err := updater.DeleteClusterRoles(selector, kubeClient); err != nil { - log.Error(err) - errors = multierror.Append(errors, fmt.Errorf("kubeCli.DeleteClusterRole error: %v", err)) + if err := updater.DeleteClusterRoles(selector, clientset); err != nil { + log.Errorf("failed to delete clusterRoles for clusterID: %s, the error is: %s", clusterID, err) + errors = multierror.Append(errors, err) } - if err := updater.DeleteClusterRoleBindings(selector, kubeClient); err != nil { - log.Error(err) - errors = multierror.Append(errors, fmt.Errorf("kubeCli.DeleteClusterRoleBinding error: %v", err)) + if err := updater.DeletePersistentVolumes(selector, clientset); err != nil { + log.Errorf("failed to delete PV for clusterID: %s, the error is: %s", clusterID, err) + errors = multierror.Append(errors, err) } return errors.ErrorOrNil() } -// 根据namespace和selector删除所有资源 -func DeleteResourcesAsync(namespace string, selector labels.Selector, kubeClient client.Client, log *zap.SugaredLogger) error { - log.Infof("[%s][%s] delete kube resources", namespace, selector) +// DeleteNamespacedResource deletes the namespaced resources by labels. +func DeleteNamespacedResource(namespace string, selector labels.Selector, clusterID string, log *zap.SugaredLogger) error { + log.Infof("Deleting namespaced resources with selector: [%s] in namespace [%s]", selector, namespace) + + clientset, err := kubeclient.GetKubeClientSet(config.HubServerAddress(), clusterID) + if err != nil { + log.Errorf("failed to create kubernetes clientset for clusterID: %s, the error is: %s", clusterID, err) + return err + } errors := new(multierror.Error) - if err := updater.DeleteDeployments(namespace, selector, kubeClient); err != nil { + if err := updater.DeleteDeployments(namespace, selector, clientset); err != nil { log.Error(err) errors = multierror.Append(errors, fmt.Errorf("kubeCli.DeleteDeployments error: %v", err)) } // could have replicas created by deployment - if err := updater.DeleteReplicaSets(namespace, selector, kubeClient); err != nil { + if err := updater.DeleteReplicaSets(namespace, selector, clientset); err != nil { log.Error(err) errors = multierror.Append(errors, fmt.Errorf("kubeCli.DeleteReplicaSets error: %v", err)) } - if err := updater.DeleteStatefulSets(namespace, selector, kubeClient); err != nil { + if err := updater.DeleteStatefulSets(namespace, selector, clientset); err != nil { log.Error(err) errors = multierror.Append(errors, fmt.Errorf("kubeCli.DeleteStatefulSets error: %v", err)) } - if err := updater.DeleteJobs(namespace, selector, kubeClient); err != nil { + if err := updater.DeleteJobs(namespace, selector, clientset); err != nil { log.Error(err) errors = multierror.Append(errors, fmt.Errorf("kubeCli.DeleteJobs error: %v", err)) } - if err := updater.DeleteServices(namespace, selector, kubeClient); err != nil { + if err := updater.DeleteServices(namespace, selector, clientset); err != nil { log.Error(err) errors = multierror.Append(errors, fmt.Errorf("kubeCli.DeleteServices error: %v", err)) } - if err := updater.DeleteIngresses(namespace, selector, kubeClient); err != nil { + // TODO: Questionable delete logic, needs further attention + if err := updater.DeleteIngresses(namespace, selector, clientset); err != nil { log.Error(err) errors = multierror.Append(errors, fmt.Errorf("kubeCli.DeleteIngresses error: %v", err)) } - if err := updater.DeleteSecrets(namespace, selector, kubeClient); err != nil { + if err := updater.DeleteSecrets(namespace, selector, clientset); err != nil { log.Error(err) errors = multierror.Append(errors, fmt.Errorf("kubeCli.DeleteSecrets error: %v", err)) } - if err := updater.DeleteConfigMaps(namespace, selector, kubeClient); err != nil { + if err := updater.DeleteConfigMaps(namespace, selector, clientset); err != nil { log.Error(err) errors = multierror.Append(errors, fmt.Errorf("kubeCli.DeleteConfigMaps error: %v", err)) } - if err := updater.DeletePersistentVolumeClaims(namespace, selector, kubeClient); err != nil { + if err := updater.DeletePersistentVolumeClaims(namespace, selector, clientset); err != nil { log.Error(err) errors = multierror.Append(errors, fmt.Errorf("kubeCli.DeletePersistentVolumeClaim error: %v", err)) } - if err := updater.DeletePersistentVolumes(namespace, selector, kubeClient); err != nil { - log.Error(err) - errors = multierror.Append(errors, fmt.Errorf("kubeCli.DeletePersistentVolume error: %v", err)) - } - - if err := updater.DeleteServiceAccounts(namespace, selector, kubeClient); err != nil { + if err := updater.DeleteServiceAccounts(namespace, selector, clientset); err != nil { log.Error(err) errors = multierror.Append(errors, fmt.Errorf("kubeCli.DeleteServiceAccounts error: %v", err)) } - if err := updater.DeleteCronJobs(namespace, selector, kubeClient); err != nil { + if err := updater.DeleteCronJobs(namespace, selector, clientset); err != nil { log.Error(err) errors = multierror.Append(errors, fmt.Errorf("kubeCli.DeleteCronJobs error: %v", err)) } - if err := updater.DeleteRoleBindings(namespace, selector, kubeClient); err != nil { + if err := updater.DeleteRoleBindings(namespace, selector, clientset); err != nil { log.Error(err) errors = multierror.Append(errors, fmt.Errorf("kubeCli.DeleteRoleBinding error: %v", err)) } - if err := updater.DeleteRoles(namespace, selector, kubeClient); err != nil { + if err := updater.DeleteRoles(namespace, selector, clientset); err != nil { log.Error(err) errors = multierror.Append(errors, fmt.Errorf("kubeCli.DeleteRole error: %v", err)) } @@ -325,6 +153,28 @@ func DeleteResourcesAsync(namespace string, selector labels.Selector, kubeClient return errors.ErrorOrNil() } +func DeleteNamespaceIfMatch(namespace string, selector labels.Selector, clusterID string, log *zap.SugaredLogger) error { + log.Infof("Checking if namespace [%s] has matching labels: [%s]", namespace, selector.String()) + + clientset, err := kubeclient.GetKubeClientSet(config.HubServerAddress(), clusterID) + if err != nil { + log.Errorf("failed to create kubernetes clientset for clusterID: %s, the error is: %s", clusterID, err) + return err + } + + ns, err := clientset.CoreV1().Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{}) + if err != nil { + log.Errorf("failed to list namespace to delete matching namespace in cluster ID: %s, the error is: %s", clusterID, err) + return err + } + + if selector.Matches(labels.Set(ns.Labels)) { + return updater.DeleteNamespace(namespace, clientset) + } + + return nil +} + func GetProductEnvNamespace(envName, productName, namespace string) string { if namespace != "" { return namespace @@ -339,3 +189,76 @@ func GetProductEnvNamespace(envName, productName, namespace string) string { } return product.Namespace } + +func GetProductTargetMap(prod *commonmodels.Product) (map[string][]commonmodels.DeployEnv, map[string]string) { + resp := make(map[string][]commonmodels.DeployEnv) + imageNameM := make(map[string]string) + if prod.Source == setting.SourceFromExternal { + services, _ := commonrepo.NewServiceColl().ListExternalWorkloadsBy(prod.ProductName, prod.EnvName) + + currentServiceNames := sets.NewString() + for _, service := range services { + currentServiceNames.Insert(service.ServiceName) + } + + servicesInExternalEnv, _ := commonrepo.NewServicesInExternalEnvColl().List(&commonrepo.ServicesInExternalEnvArgs{ + ProductName: prod.ProductName, + EnvName: prod.EnvName, + }) + + externalServiceNames := sets.NewString() + for _, serviceInExternalEnv := range servicesInExternalEnv { + if !currentServiceNames.Has(serviceInExternalEnv.ServiceName) { + externalServiceNames.Insert(serviceInExternalEnv.ServiceName) + } + } + + if len(externalServiceNames) > 0 { + newServices, _ := commonrepo.NewServiceColl().ListExternalWorkloadsBy(prod.ProductName, "", externalServiceNames.List()...) + for _, service := range newServices { + services = append(services, service) + } + } + + for _, service := range services { + for _, container := range service.Containers { + env := service.ServiceName + "/" + container.Name + deployEnv := commonmodels.DeployEnv{Type: setting.K8SDeployType, Env: env} + target := strings.Join([]string{service.ProductName, service.ServiceName, container.Name}, SplitSymbol) + resp[target] = append(resp[target], deployEnv) + + imageNameM[target] = util.GetImageNameFromContainerInfo(container.ImageName, container.Name) + } + } + return resp, imageNameM + } + for _, services := range prod.Services { + for _, serviceObj := range services { + switch serviceObj.Type { + case setting.K8SDeployType: + for _, container := range serviceObj.Containers { + env := serviceObj.ServiceName + "/" + container.Name + deployEnv := commonmodels.DeployEnv{Type: setting.K8SDeployType, Env: env} + target := strings.Join([]string{serviceObj.ProductName, serviceObj.ServiceName, container.Name}, SplitSymbol) + resp[target] = append(resp[target], deployEnv) + + imageNameM[target] = util.GetImageNameFromContainerInfo(container.ImageName, container.Name) + } + case setting.PMDeployType: + deployEnv := commonmodels.DeployEnv{Type: setting.PMDeployType, Env: serviceObj.ServiceName} + target := strings.Join([]string{serviceObj.ProductName, serviceObj.ServiceName, serviceObj.ServiceName}, SplitSymbol) + resp[target] = append(resp[target], deployEnv) + case setting.HelmDeployType: + for _, container := range serviceObj.Containers { + env := serviceObj.ServiceName + "/" + container.Name + deployEnv := commonmodels.DeployEnv{Type: setting.HelmDeployType, Env: env} + target := strings.Join([]string{serviceObj.ProductName, serviceObj.ServiceName, container.Name}, SplitSymbol) + resp[target] = append(resp[target], deployEnv) + + imageNameM[target] = util.GetImageNameFromContainerInfo(container.ImageName, container.Name) + } + } + } + } + return resp, imageNameM +} diff --git a/pkg/microservice/aslan/core/common/service/registry.go b/pkg/microservice/aslan/core/common/service/registry.go index ecf5a1c37500577c5da506c5071f7e482062ded2..9b7b07980712918c9e90a889b9157bebb75f25ba 100644 --- a/pkg/microservice/aslan/core/common/service/registry.go +++ b/pkg/microservice/aslan/core/common/service/registry.go @@ -17,8 +17,17 @@ limitations under the License. package service import ( + "encoding/base64" + "errors" "fmt" - + "strings" + "sync" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ecr" "go.uber.org/zap" "sigs.k8s.io/controller-runtime/pkg/client" @@ -26,71 +35,196 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" + "github.com/koderover/zadig/pkg/tool/crypto" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/util" ) -func FindDefaultRegistry(log *zap.SugaredLogger) (*models.RegistryNamespace, error) { +var expirationTime = 10 * time.Hour + +var awsKeyMap sync.Map + +type awsKeyWithExpiration struct { + AccessKey string + SecretKey string + Expiration int64 +} + +func (k *awsKeyWithExpiration) IsExpired() bool { + return time.Now().Unix() > k.Expiration +} + +func FindRegistryById(registryId string, getRealCredential bool, log *zap.SugaredLogger) (reg *models.RegistryNamespace, isSystemDefault bool, err error) { + return findRegisty(&mongodb.FindRegOps{ID: registryId}, getRealCredential, log) +} + +func findRegisty(regOps *mongodb.FindRegOps, getRealCredential bool, log *zap.SugaredLogger) (reg *models.RegistryNamespace, isSystemDefault bool, err error) { // TODO: 多租户适配 - resp, err := mongodb.NewRegistryNamespaceColl().Find(&mongodb.FindRegOps{ - IsDefault: true, - }) + resp, err := mongodb.NewRegistryNamespaceColl().Find(regOps) + isSystemDefault = false if err != nil { - log.Warnf("RegistryNamespace.Find error: %v", err) + log.Warnf("RegistryNamespace.Find error: %s", err) resp = &models.RegistryNamespace{ RegAddr: config.RegistryAddress(), AccessKey: config.RegistryAccessKey(), SecretKey: config.RegistrySecretKey(), Namespace: config.RegistryNamespace(), } + isSystemDefault = true } - ak := resp.AccessKey - sk := resp.SecretKey - if resp.RegProvider == config.SWRProvider { - ak = fmt.Sprintf("%s@%s", resp.Region, resp.AccessKey) - sk = util.ComputeHmacSha256(resp.AccessKey, resp.SecretKey) + if !getRealCredential { + return resp, isSystemDefault, nil + } + switch resp.RegProvider { + case config.RegistryTypeSWR: + resp.SecretKey = util.ComputeHmacSha256(resp.AccessKey, resp.SecretKey) + resp.AccessKey = fmt.Sprintf("%s@%s", resp.Region, resp.AccessKey) + case config.RegistryTypeAWS: + realAK, realSK, err := getAWSRegistryCredential(resp.ID.Hex(), resp.AccessKey, resp.SecretKey, resp.Region) + if err != nil { + log.Errorf("Failed to get keypair from aws, the error is: %s", err) + return nil, isSystemDefault, err + } + resp.AccessKey = realAK + resp.SecretKey = realSK } - resp.AccessKey = ak - resp.SecretKey = sk - return resp, nil + return resp, isSystemDefault, nil } -func GetDefaultRegistryNamespace(log *zap.SugaredLogger) (*models.RegistryNamespace, error) { - resp, err := mongodb.NewRegistryNamespaceColl().Find(&mongodb.FindRegOps{IsDefault: true}) - if err != nil { - log.Errorf("get default registry error: %v", err) - return resp, fmt.Errorf("get default registry error: %v", err) - } - return resp, nil +func FindDefaultRegistry(getRealCredential bool, log *zap.SugaredLogger) (reg *models.RegistryNamespace, isSystemDefault bool, err error) { + return findRegisty(&mongodb.FindRegOps{IsDefault: true}, getRealCredential, log) } -func ListRegistryNamespaces(log *zap.SugaredLogger) ([]*models.RegistryNamespace, error) { +func ListRegistryNamespaces(encryptedKey string, getRealCredential bool, log *zap.SugaredLogger) ([]*models.RegistryNamespace, error) { resp, err := mongodb.NewRegistryNamespaceColl().FindAll(&mongodb.FindRegOps{}) if err != nil { - log.Errorf("RegistryNamespace.List error: %v", err) - return resp, fmt.Errorf("RegistryNamespace.List error: %v", err) + log.Errorf("RegistryNamespace.List error: %s", err) + return resp, fmt.Errorf("RegistryNamespace.List error: %s", err) + } + var aesKey *GetAesKeyFromEncryptedKeyResp + if len(encryptedKey) > 0 { + aesKey, err = GetAesKeyFromEncryptedKey(encryptedKey, log) + if err != nil { + log.Errorf("RegistryNamespace.List GetAesKeyFromEncryptedKey error: %s", err) + return nil, err + } + } + if !getRealCredential { + if len(encryptedKey) > 0 { + for _, reg := range resp { + reg.SecretKey, err = crypto.AesEncryptByKey(reg.SecretKey, aesKey.PlainText) + if err != nil { + log.Errorf("RegistryNamespace.List AesEncryptByKey error: %s", err) + return nil, err + } + } + } + return resp, nil + } + + for _, reg := range resp { + switch reg.RegProvider { + case config.RegistryTypeSWR: + reg.SecretKey = util.ComputeHmacSha256(reg.AccessKey, reg.SecretKey) + reg.AccessKey = fmt.Sprintf("%s@%s", reg.Region, reg.AccessKey) + case config.RegistryTypeAWS: + realAK, realSK, err := getAWSRegistryCredential(reg.ID.Hex(), reg.AccessKey, reg.SecretKey, reg.Region) + if err != nil { + log.Errorf("Failed to get keypair from aws, the error is: %s", err) + return nil, err + } + reg.AccessKey = realAK + reg.SecretKey = realSK + } + if len(encryptedKey) == 0 { + continue + } + reg.SecretKey, err = crypto.AesEncryptByKey(reg.SecretKey, aesKey.PlainText) + if err != nil { + log.Errorf("RegistryNamespace.List AesEncryptByKey error: %s", err) + return nil, err + } } return resp, nil } -func EnsureDefaultRegistrySecret(namespace string, kubeClient client.Client, log *zap.SugaredLogger) error { - reg, err := FindDefaultRegistry(log) - if err != nil { - log.Errorf( - "service.EnsureRegistrySecret: failed to find default candidate registry: %s %v", - namespace, err, - ) - return err +func EnsureDefaultRegistrySecret(namespace string, registryId string, kubeClient client.Client, log *zap.SugaredLogger) error { + var reg *models.RegistryNamespace + var err error + if len(registryId) > 0 { + reg, _, err = FindRegistryById(registryId, true, log) + if err != nil { + log.Errorf( + "service.EnsureRegistrySecret: failed to find registry: %s error msg:%s", + registryId, err, + ) + return err + } + } else { + reg, _, err = FindDefaultRegistry(true, log) + if err != nil { + log.Errorf( + "service.EnsureRegistrySecret: failed to find default candidate registry: %s %s", + namespace, err, + ) + return err + } } - err = kube.CreateOrUpdateRegistrySecret(namespace, reg, kubeClient) + err = kube.CreateOrUpdateDefaultRegistrySecret(namespace, reg, kubeClient) if err != nil { - log.Errorf("[%s] CreateDockerSecret error: %v", namespace, err) + log.Errorf("[%s] CreateDockerSecret error: %s", namespace, err) return e.ErrUpdateSecret.AddDesc(e.CreateDefaultRegistryErrMsg) } return nil } + +func getAWSRegistryCredential(id, ak, sk, region string) (realAK string, realSK string, err error) { + // first we try to get ak/sk from our memory cache + obj, ok := awsKeyMap.Load(id) + if ok { + keypair, ok := obj.(awsKeyWithExpiration) + if ok { + if !keypair.IsExpired() { + return keypair.AccessKey, keypair.SecretKey, nil + } + } + } + creds := credentials.NewStaticCredentials(ak, sk, "") + config := &aws.Config{ + Region: aws.String(region), + Credentials: creds, + } + sess, err := session.NewSession(config) + if err != nil { + return "", "", err + } + svc := ecr.New(sess) + input := &ecr.GetAuthorizationTokenInput{} + + result, err := svc.GetAuthorizationToken(input) + if err != nil { + return "", "", err + } + // since the new AWS ECR will give a token that has access to ALL the repository, we use the first token + encodedToken := *result.AuthorizationData[0].AuthorizationToken + rawDecodedText, err := base64.StdEncoding.DecodeString(encodedToken) + if err != nil { + return "", "", err + } + keypair := strings.Split(string(rawDecodedText), ":") + if len(keypair) != 2 { + return "", "", errors.New("format of keypair is invalid") + } + // cache the aws ak/sk + awsKeyMap.Store(id, awsKeyWithExpiration{ + AccessKey: keypair[0], + SecretKey: keypair[1], + Expiration: time.Now().Add(expirationTime).Unix(), + }) + return keypair[0], keypair[1], nil +} diff --git a/pkg/microservice/aslan/core/common/service/registry/service.go b/pkg/microservice/aslan/core/common/service/registry/service.go index 91fb00b097cd6055cff9cae725ebdcddd8190764..77c0a8487b4e463a2c2508fca16cb9041d974aa4 100644 --- a/pkg/microservice/aslan/core/common/service/registry/service.go +++ b/pkg/microservice/aslan/core/common/service/registry/service.go @@ -18,6 +18,8 @@ package registry import ( "context" + "crypto/tls" + "crypto/x509" "encoding/json" "fmt" "net" @@ -28,6 +30,10 @@ import ( "sync" "time" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ecr" "github.com/docker/distribution" "github.com/docker/distribution/manifest/schema2" "github.com/docker/distribution/reference" @@ -75,16 +81,24 @@ type Service interface { GetImageInfo(option GetRepoImageDetailOption, log *zap.SugaredLogger) (*commonmodels.DeliveryImage, error) } -func NewV2Service(provider string) Service { +func NewV2Service(provider string, tlsEnabled bool, tlsCert string) Service { + // since SWR & AWS services are provided by known provider, we assume it is signed by well known CA switch provider { - case config.SWRProvider: - return &SwrService{} + case config.RegistryTypeSWR: + return &swrService{} + case config.RegistryTypeAWS: + return &ecrService{} default: - return &v2RegistryService{} + return &v2RegistryService{ + EnableHTTPS: tlsEnabled, + CustomCert: tlsCert, + } } } type v2RegistryService struct { + EnableHTTPS bool + CustomCert string } type authClient struct { @@ -109,10 +123,21 @@ func (s *v2RegistryService) createClient(ep Endpoint, logger *zap.SugaredLogger) KeepAlive: 30 * time.Second, } + tlsConfig := &tls.Config{} + + if s.EnableHTTPS && s.CustomCert != "" { + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM([]byte(s.CustomCert)) + tlsConfig.RootCAs = caCertPool + } else if !s.EnableHTTPS { + tlsConfig.InsecureSkipVerify = true + } + base := &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: direct.DialContext, TLSHandshakeTimeout: 10 * time.Second, + TLSClientConfig: tlsConfig, DisableKeepAlives: true, } @@ -356,10 +381,10 @@ func (s *v2RegistryService) ListRepoImages(option ListRepoImagesOption, log *zap return resp, nil } -type SwrService struct { +type swrService struct { } -func (s *SwrService) createClient(ep Endpoint) (cli *swr.SwrClient) { +func (s *swrService) createClient(ep Endpoint) (cli *swr.SwrClient) { endpoint := fmt.Sprintf("https://swr-api.%s.myhuaweicloud.com", ep.Region) auth := basic.NewCredentialsBuilder(). WithAk(ep.Ak). @@ -374,7 +399,7 @@ func (s *SwrService) createClient(ep Endpoint) (cli *swr.SwrClient) { return client } -func (s *SwrService) ListRepoImages(option ListRepoImagesOption, log *zap.SugaredLogger) (resp *ReposResp, err error) { +func (s *swrService) ListRepoImages(option ListRepoImagesOption, log *zap.SugaredLogger) (resp *ReposResp, err error) { swrCli := s.createClient(option.Endpoint) var wg wait.Group @@ -429,7 +454,7 @@ func (s *SwrService) ListRepoImages(option ListRepoImagesOption, log *zap.Sugare } -func (s *SwrService) GetImageInfo(option GetRepoImageDetailOption, log *zap.SugaredLogger) (di *commonmodels.DeliveryImage, err error) { +func (s *swrService) GetImageInfo(option GetRepoImageDetailOption, log *zap.SugaredLogger) (di *commonmodels.DeliveryImage, err error) { swrCli := s.createClient(option.Endpoint) request := &model.ListRepositoryTagsRequest{Tag: &option.Tag, Namespace: option.Namespace, Repository: option.Image} @@ -451,3 +476,104 @@ func (s *SwrService) GetImageInfo(option GetRepoImageDetailOption, log *zap.Suga return &commonmodels.DeliveryImage{}, nil } + +type ecrService struct { +} + +func (s *ecrService) getECRService(ep Endpoint, log *zap.SugaredLogger) (*ecr.ECR, error) { + creds := credentials.NewStaticCredentials(ep.Ak, ep.Sk, "") + config := &aws.Config{ + Region: aws.String(ep.Region), + Credentials: creds, + } + sess, err := session.NewSession(config) + if err != nil { + log.Errorf("Failed to create aws session, err: %s", err) + return nil, err + } + return ecr.New(sess), nil +} + +func (s *ecrService) ListRepoImages(option ListRepoImagesOption, log *zap.SugaredLogger) (resp *ReposResp, err error) { + svc, err := s.getECRService(option.Endpoint, log) + if err != nil { + return nil, err + } + + var wg wait.Group + var mutex sync.RWMutex + resp = &ReposResp{Total: len(option.Repos)} + resultChan := make(chan *Repo) + defer close(resultChan) + + for _, repo := range option.Repos { + name := repo + wg.Start(func() { + input := &ecr.ListImagesInput{ + RepositoryName: aws.String(name), + } + result, err := svc.ListImages(input) + if err != nil { + log.Errorf("Failed to get image information from aws, error: %s", err) + return + } + var koderoverTags, customTags, sortedTags []string + for _, image := range result.ImageIds { + tagArray := strings.Split(*image.ImageTag, "-") + if len(tagArray) > 1 && len(tagArray[0]) == 14 { + if _, err := time.Parse("20060102150405", tagArray[0]); err == nil { + koderoverTags = append(koderoverTags, *image.ImageTag) + continue + } + } + customTags = append(customTags, *image.ImageTag) + } + + sort.Sort(sort.Reverse(sort.StringSlice(koderoverTags))) + sortedTags = append(koderoverTags, customTags...) + + mutex.Lock() + resp.Repos = append(resp.Repos, &Repo{ + Name: name, + Tags: sortedTags, + }) + mutex.Unlock() + }) + + } + wg.Wait() + + return resp, nil +} + +func (s *ecrService) GetImageInfo(option GetRepoImageDetailOption, log *zap.SugaredLogger) (di *commonmodels.DeliveryImage, err error) { + svc, err := s.getECRService(option.Endpoint, log) + if err != nil { + return nil, err + } + input := &ecr.DescribeImagesInput{ + ImageIds: []*ecr.ImageIdentifier{ + { + ImageTag: aws.String(option.Tag), + }, + }, + RepositoryName: aws.String(option.Image), + } + result, err := svc.DescribeImages(input) + if err != nil { + err = errors.Wrapf(err, "failed to get image info of %s:%s", option.Image, option.Tag) + return + } + // since only one image tag is passed, only one image detail will be in this detail list + // so only the first one will be used + for _, imageDetail := range result.ImageDetails { + return &commonmodels.DeliveryImage{ + RepoName: option.Image, + TagName: option.Tag, + CreationTime: imageDetail.ImagePushedAt.String(), + ImageDigest: *imageDetail.ImageDigest, + ImageSize: *imageDetail.ImageSizeInBytes, + }, nil + } + return &commonmodels.DeliveryImage{}, nil +} diff --git a/pkg/microservice/aslan/core/common/service/render.go b/pkg/microservice/aslan/core/common/service/render.go index 3552b1e1bf28b9e3bffbb7f6c36cb4aa1e60fb59..3b4c2f9a7ee8a3d8eba72e5ed158419aca1e5f62 100644 --- a/pkg/microservice/aslan/core/common/service/render.go +++ b/pkg/microservice/aslan/core/common/service/render.go @@ -20,12 +20,14 @@ import ( "encoding/json" "errors" "fmt" + "reflect" "sort" "strings" "go.uber.org/zap" "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" templatemodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" @@ -33,32 +35,48 @@ import ( "github.com/koderover/zadig/pkg/setting" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" + yamlutil "github.com/koderover/zadig/pkg/util/yaml" ) type RepoConfig struct { CodehostID int `json:"codehostID,omitempty"` Owner string `json:"owner,omitempty"` + Namespace string `json:"namespace,omitempty"` Repo string `json:"repo,omitempty"` Branch string `json:"branch,omitempty"` ValuesPaths []string `json:"valuesPaths,omitempty"` } type KVPair struct { - Key string `json:"key"` - Value string `json:"value"` + Key string `json:"key"` + Value interface{} `json:"value"` +} + +type ValuesDataArgs struct { + YamlSource string `json:"yamlSource,omitempty"` + AutoSync bool `json:"autoSync,omitempty"` + GitRepoConfig *RepoConfig `json:"gitRepoConfig,omitempty"` } type RenderChartArg struct { - EnvName string `json:"envName,omitempty"` - ServiceName string `json:"serviceName,omitempty"` - ChartVersion string `json:"chartVersion,omitempty"` - YamlSource string `json:"yamlSource,omitempty"` - GitRepoConfig *RepoConfig `json:"gitRepoConfig,omitempty"` - OverrideValues []*KVPair `json:"overrideValues,omitempty"` - ValuesYAML string `json:"valuesYAML,omitempty"` + EnvName string `json:"envName,omitempty"` + ServiceName string `json:"serviceName,omitempty"` + ChartVersion string `json:"chartVersion,omitempty"` + OverrideValues []*KVPair `json:"overrideValues,omitempty"` + OverrideYaml string `json:"overrideYaml,omitempty"` + ValuesData *ValuesDataArgs `json:"valuesData,omitempty"` + YamlData *templatemodels.CustomYaml `json:"yaml_data,omitempty"` } -func (args *RenderChartArg) toOverrideValueString() string { +type RenderChartDiffResult string + +const ( + Different RenderChartDiffResult = "different" + Same RenderChartDiffResult = "same" + LogicSame RenderChartDiffResult = "logicSame" +) + +func (args *RenderChartArg) ToOverrideValueString() string { if len(args.OverrideValues) == 0 { return "" } @@ -79,61 +97,49 @@ func (args *RenderChartArg) fromOverrideValueString(valueStr string) { args.OverrideValues = make([]*KVPair, 0) err := json.Unmarshal([]byte(valueStr), &args.OverrideValues) if err != nil { - log.Errorf("decode override value fail, ") + log.Errorf("decode override value fail, err: %s", err) } } func (args *RenderChartArg) toCustomValuesYaml() *templatemodels.CustomYaml { - switch args.YamlSource { - case setting.ValuesYamlSourceFreeEdit: - return &templatemodels.CustomYaml{ - YamlSource: args.YamlSource, - YamlContent: args.ValuesYAML, - } - case setting.ValuesYamlSourceGitRepo: - return &templatemodels.CustomYaml{ - YamlSource: args.YamlSource, - YamlContent: args.ValuesYAML, - ValuesPaths: args.GitRepoConfig.ValuesPaths, - GitRepoConfig: &templatemodels.GitRepoConfig{ - CodehostID: args.GitRepoConfig.CodehostID, - Owner: args.GitRepoConfig.Owner, - Repo: args.GitRepoConfig.Repo, - Branch: args.GitRepoConfig.Branch, - }, + ret := &templatemodels.CustomYaml{ + YamlContent: args.OverrideYaml, + } + if args.ValuesData != nil { + ret.Source = args.ValuesData.YamlSource + ret.AutoSync = args.ValuesData.AutoSync + if args.ValuesData.GitRepoConfig != nil { + repoData := &models.CreateFromRepo{ + GitRepoConfig: &templatemodels.GitRepoConfig{ + CodehostID: args.ValuesData.GitRepoConfig.CodehostID, + Owner: args.ValuesData.GitRepoConfig.Owner, + Namespace: args.ValuesData.GitRepoConfig.Namespace, + Repo: args.ValuesData.GitRepoConfig.Repo, + Branch: args.ValuesData.GitRepoConfig.Branch, + }, + } + if len(args.ValuesData.GitRepoConfig.ValuesPaths) > 0 { + repoData.LoadPath = args.ValuesData.GitRepoConfig.ValuesPaths[0] + } + ret.SourceDetail = repoData + ret.Source = setting.SourceFromGitRepo } } - return nil + return ret } func (args *RenderChartArg) fromCustomValueYaml(customValuesYaml *templatemodels.CustomYaml) { if customValuesYaml == nil { return } - args.YamlSource = customValuesYaml.YamlSource - switch customValuesYaml.YamlSource { - case setting.ValuesYamlSourceFreeEdit: - args.ValuesYAML = customValuesYaml.YamlContent - case setting.ValuesYamlSourceGitRepo: - args.ValuesYAML = "" - if customValuesYaml.GitRepoConfig != nil { - args.GitRepoConfig = &RepoConfig{ - CodehostID: customValuesYaml.GitRepoConfig.CodehostID, - Owner: customValuesYaml.GitRepoConfig.Owner, - Repo: customValuesYaml.GitRepoConfig.Repo, - Branch: customValuesYaml.GitRepoConfig.Branch, - ValuesPaths: customValuesYaml.ValuesPaths, - } - } - - } + args.OverrideYaml = customValuesYaml.YamlContent } // FillRenderChartModel fill render chart model func (args *RenderChartArg) FillRenderChartModel(chart *templatemodels.RenderChart, version string) { chart.ServiceName = args.ServiceName chart.ChartVersion = version - chart.OverrideValues = args.toOverrideValueString() + chart.OverrideValues = args.ToOverrideValueString() chart.OverrideYaml = args.toCustomValuesYaml() } @@ -145,6 +151,36 @@ func (args *RenderChartArg) LoadFromRenderChartModel(chart *templatemodels.Rende args.fromCustomValueYaml(chart.OverrideYaml) } +func (args *RenderChartArg) GetUniqueKvMap() map[string]interface{} { + uniqueKvs := make(map[string]interface{}) + for index := range args.OverrideValues { + kv := args.OverrideValues[len(args.OverrideValues)-index-1] + if _, ok := uniqueKvs[kv.Key]; ok { + continue + } + uniqueKvs[kv.Key] = kv.Value + } + return uniqueKvs +} + +// DiffValues generate diff values to override from two chart args +func (args *RenderChartArg) DiffValues(target *RenderChartArg) RenderChartDiffResult { + argsUniqueKvs := args.GetUniqueKvMap() + targetUniqueKvs := target.GetUniqueKvMap() + if len(argsUniqueKvs) != len(targetUniqueKvs) || !reflect.DeepEqual(argsUniqueKvs, targetUniqueKvs) { + return Different + } + // override yamls have the same text content + if args.OverrideYaml == target.OverrideYaml { + return Same + } + equal, _ := yamlutil.Equal(args.OverrideYaml, target.OverrideYaml) + if equal { + return LogicSame + } + return Different +} + func listTmplRenderKeys(productTmplName string, log *zap.SugaredLogger) ([]*templatemodels.RenderKV, map[string]*templatemodels.ServiceInfo, error) { //如果没找到对应产品，则kv为空 prodTmpl, err := template.NewProductColl().Find(productTmplName) @@ -158,14 +194,16 @@ func listTmplRenderKeys(productTmplName string, log *zap.SugaredLogger) ([]*temp return kvs, prodTmpl.AllServiceInfoMap(), err } -func GetRenderSet(renderName string, revision int64, log *zap.SugaredLogger) (*commonmodels.RenderSet, error) { +func GetRenderSet(renderName string, revision int64, isDefault bool, envName string, log *zap.SugaredLogger) (*commonmodels.RenderSet, error) { // 未指定renderName返回空的renderSet if renderName == "" { return &commonmodels.RenderSet{KVs: []*templatemodels.RenderKV{}}, nil } opt := &commonrepo.RenderSetFindOption{ - Name: renderName, - Revision: revision, + Name: renderName, + Revision: revision, + IsDefault: isDefault, + EnvName: envName, } resp, found, err := commonrepo.NewRenderSetColl().FindRenderSet(opt) if err != nil { @@ -195,29 +233,25 @@ func GetRenderSetInfo(renderName string, revision int64) (*commonmodels.RenderSe } // ValidateRenderSet 检查指定renderSet是否能覆盖产品所有需要渲染的值 -func ValidateRenderSet(productName, renderName string, serviceInfo *templatemodels.ServiceInfo, log *zap.SugaredLogger) (*commonmodels.RenderSet, error) { +func ValidateRenderSet(productName, renderName, envName string, serviceInfo *templatemodels.ServiceInfo, log *zap.SugaredLogger) (*commonmodels.RenderSet, error) { resp := &commonmodels.RenderSet{ProductTmpl: productName} var err error if renderName != "" { - opt := &commonrepo.RenderSetFindOption{Name: renderName} + opt := &commonrepo.RenderSetFindOption{Name: renderName, ProductTmpl: productName, EnvName: envName} resp, err = commonrepo.NewRenderSetColl().Find(opt) if err != nil { log.Errorf("find renderset[%s] error: %v", renderName, err) return resp, err } } - if renderName != "" && resp.ProductTmpl != productName { - log.Errorf("renderset[%s] not match product[%s]", renderName, productName) - return resp, fmt.Errorf("renderset[%s] not match product[%s]", renderName, productName) - } if serviceInfo == nil { - if err := IsAllKeyCovered(resp, log); err != nil { - log.Errorf("[%s]cover all key [%s] error: %v", productName, renderName, err) - return resp, err - } + //if err := IsAllKeyCovered(resp, log); err != nil { + // log.Errorf("[%s]cover all key [%s] error: %v", productName, renderName, err) + // return resp, err + //} } else { // 单个服务是否全覆盖判断 - if err := IsAllKeyCoveredService(serviceInfo.Owner, serviceInfo.Name, resp, log); err != nil { + if err := IsAllKeyCoveredService(serviceInfo.Owner, serviceInfo.Name, resp); err != nil { log.Errorf("[%s]cover all key [%s] error: %v", productName, renderName, err) return resp, err } @@ -225,12 +259,61 @@ func ValidateRenderSet(productName, renderName string, serviceInfo *templatemode return resp, nil } +func mergeKVs(newKVs []*templatemodels.RenderKV, oldKVs []*templatemodels.RenderKV) []*templatemodels.RenderKV { + var result []*templatemodels.RenderKV + newKVsMap := make(map[string]*templatemodels.RenderKV) + for _, v := range newKVs { + newKVsMap[v.Key] = v + } + oldKVsMap := make(map[string]*templatemodels.RenderKV) + for _, v := range oldKVs { + oldKVsMap[v.Key] = v + if newKV, ok := newKVsMap[v.Key]; ok { + result = append(result, newKV) + continue + } + result = append(result, v) + } + for _, v := range newKVs { + if _, ok := oldKVsMap[v.Key]; !ok { + result = append(result, v) + } + } + return result +} + +func CreateRenderSetByMerge(args *commonmodels.RenderSet, log *zap.SugaredLogger) error { + opt := &commonrepo.RenderSetFindOption{Name: args.Name, ProductTmpl: args.ProductTmpl, EnvName: args.EnvName} + rs, err := commonrepo.NewRenderSetColl().Find(opt) + if rs != nil && err == nil { + if rs.Diff(args) { + args.IsDefault = rs.IsDefault + } else { + return nil + } + mergedKVs := mergeKVs(args.KVs, rs.KVs) + args.KVs = mergedKVs + } + if err := ensureRenderSetArgs(args); err != nil { + log.Error(err) + return e.ErrCreateRenderSet.AddDesc(err.Error()) + } + if err := commonrepo.NewRenderSetColl().Create(args); err != nil { + errMsg := fmt.Sprintf("[RenderSet.Create] %s error: %v", args.Name, err) + log.Error(errMsg) + return e.ErrCreateRenderSet.AddDesc(errMsg) + } + return nil +} + func CreateRenderSet(args *commonmodels.RenderSet, log *zap.SugaredLogger) error { - opt := &commonrepo.RenderSetFindOption{Name: args.Name} + opt := &commonrepo.RenderSetFindOption{ + Name: args.Name, + ProductTmpl: args.ProductTmpl, + EnvName: args.EnvName, + } rs, err := commonrepo.NewRenderSetColl().Find(opt) if rs != nil && err == nil { - // 已经存在渲染配置集 - // 判断是否有修改 if rs.Diff(args) { args.IsDefault = rs.IsDefault } else { @@ -251,12 +334,16 @@ func CreateRenderSet(args *commonmodels.RenderSet, log *zap.SugaredLogger) error // CreateHelmRenderSet 添加renderSet func CreateHelmRenderSet(args *commonmodels.RenderSet, log *zap.SugaredLogger) error { - opt := &commonrepo.RenderSetFindOption{Name: args.Name} + opt := &commonrepo.RenderSetFindOption{ + Name: args.Name, + ProductTmpl: args.ProductTmpl, + EnvName: args.EnvName, + } rs, err := commonrepo.NewRenderSetColl().Find(opt) if rs != nil && err == nil { // 已经存在渲染配置集 // 判断是否有修改 - if rs.HelmRenderDiff(args) { + if rs.DefaultValues != args.DefaultValues || rs.HelmRenderDiff(args) || !reflect.DeepEqual(rs.YamlData, args.YamlData) { args.IsDefault = rs.IsDefault } else { return nil @@ -447,7 +534,31 @@ func RenderValueForString(origin string, rs *commonmodels.RenderSet) string { } rs.SetKVAlias() for _, v := range rs.KVs { - origin = strings.Replace(origin, v.Alias, v.Value, -1) + if v.State == "unused" { + continue + } + origin = replaceAliasValue(origin, v) + } + return origin +} + +func replaceAliasValue(origin string, v *templatemodels.RenderKV) string { + for { + idx := strings.Index(origin, v.Alias) + if idx < 0 { + break + } + spaces := "" + start := 0 + for i := idx - 1; i >= 0; i-- { + if string(origin[i]) != " " { + break + } + start++ + } + spaces = origin[idx-start : idx] + valueStr := strings.Replace(v.Value, "\n", fmt.Sprintf("%s%s", "\n", spaces), -1) + origin = strings.Replace(origin, v.Alias, valueStr, 1) } return origin } @@ -467,8 +578,9 @@ func getValueFromSharedRenderSet(kv *templatemodels.RenderKV, productName string } renderSetOpt := &commonrepo.RenderSetFindOption{ - Name: targetProduct, - Revision: 0, + Name: targetProduct, + IsDefault: true, + Revision: 0, } renderSet, err := commonrepo.NewRenderSetColl().Find(renderSetOpt) if err != nil { @@ -526,7 +638,7 @@ func IsAllKeyCovered(arg *commonmodels.RenderSet, log *zap.SugaredLogger) error } // IsAllKeyCoveredService 检查是否覆盖所有服务key -func IsAllKeyCoveredService(productName, serviceName string, arg *commonmodels.RenderSet, log *zap.SugaredLogger) error { +func IsAllKeyCoveredService(productName, serviceName string, arg *commonmodels.RenderSet) error { opt := &commonrepo.ServiceFindOption{ ServiceName: serviceName, ProductName: productName, @@ -560,10 +672,11 @@ func ensureRenderSetArgs(args *commonmodels.RenderSet) error { if len(args.Name) == 0 { return errors.New("empty render set name") } - log := log.SugaredLogger() - if err := IsAllKeyCovered(args, log); err != nil { - return fmt.Errorf("[RenderSet.Create] %s error: %v", args.Name, err) - } + + //log := log.SugaredLogger() + //if err := IsAllKeyCovered(args, log); err != nil { + // return fmt.Errorf("[RenderSet.Create] %s error: %v", args.Name, err) + //} // 设置新的版本号 rev, err := commonrepo.NewCounterColl().GetNextSeq("renderset:" + args.Name) diff --git a/pkg/microservice/aslan/core/common/service/rsa_key.go b/pkg/microservice/aslan/core/common/service/rsa_key.go new file mode 100644 index 0000000000000000000000000000000000000000..97a1ee43c64a6de37ec119aedaea02d419640e93 --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/rsa_key.go @@ -0,0 +1,91 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "context" + "encoding/base64" + "net/url" + "strings" + + "go.uber.org/zap" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" + "github.com/koderover/zadig/pkg/tool/rsa" +) + +func GetRSAKey() ([]byte, []byte, error) { + clientset, err := kubeclient.GetKubeClientSet(config.HubServerAddress(), setting.LocalClusterID) + if err != nil { + return nil, nil, err + } + res, err := clientset.CoreV1().Secrets(config.Namespace()).Get(context.TODO(), setting.RSASecretName, metav1.GetOptions{}) + if err != nil { + return nil, nil, err + } + return res.Data["publicKey"], res.Data["privateKey"], nil +} + +type GetRSAPublicKeyRes struct { + PublicKey string `json:"publicKey"` +} + +func GetRSAPublicKey() (*GetRSAPublicKeyRes, error) { + publicKey, _, err := GetRSAKey() + if err != nil { + return nil, err + } + return &GetRSAPublicKeyRes{ + PublicKey: string(publicKey), + }, nil +} + +type GetAesKeyFromEncryptedKeyResp struct { + PlainText string `json:"plain_text"` +} + +func GetAesKeyFromEncryptedKey(encryptedKey string, log *zap.SugaredLogger) (*GetAesKeyFromEncryptedKeyResp, error) { + _, privateKey, err := GetRSAKey() + if err != nil { + log.Errorf("getAesKeyFromEncryptedKey getRSAKey error msg:%s", err) + return nil, err + } + decodedKey, err := url.QueryUnescape(encryptedKey) + if err != nil { + log.Errorf("GetAesKeyFromEncryptedKey QueryUnescape error:%s", err) + return nil, err + } + decodedKey = strings.ReplaceAll(decodedKey, " ", "+") + byteKey, err := base64.StdEncoding.DecodeString(decodedKey) + if err != nil { + log.Errorf("getAesKeyFromEncryptedKey decodeString error msg:%s", err) + return nil, err + } + + aesKey, err := rsa.DecryptByPrivateKey(byteKey, privateKey) + if err != nil { + log.Errorf("getAesKeyFromEncryptedKey decryptByPrivateKey error msg:%s", err) + return nil, err + } + + return &GetAesKeyFromEncryptedKeyResp{ + PlainText: string(aesKey), + }, nil +} diff --git a/pkg/microservice/aslan/core/common/service/s3/s3.go b/pkg/microservice/aslan/core/common/service/s3/s3.go index 24655148c8206bbc02653904d65c13b2e81f8581..e3381f24d2e0362f91a4ae06db3bcc434fdf956f 100644 --- a/pkg/microservice/aslan/core/common/service/s3/s3.go +++ b/pkg/microservice/aslan/core/common/service/s3/s3.go @@ -126,6 +126,14 @@ func (s *S3) Validate() error { return errors.New("required field is missing") } + defaultStorage, err := FindDefaultS3() + if err != nil { + return fmt.Errorf("failed to find default object storage: %s", err) + } + if s.ID == defaultStorage.ID && !s.IsDefault { + return errors.New("current storage is default and a default object storage must be set") + } + return nil } @@ -137,7 +145,7 @@ func FindDefaultS3() (*S3, error) { S3Storage: &models.S3Storage{ Ak: config.S3StorageAK(), Sk: config.S3StorageSK(), - Endpoint: config.S3StorageEndpoint(), + Endpoint: getEndpoint(), Bucket: config.S3StorageBucket(), Insecure: config.S3StorageProtocol() == "http", Provider: setting.ProviderSourceSystemDefault, @@ -148,12 +156,29 @@ func FindDefaultS3() (*S3, error) { return &S3{S3Storage: storage}, nil } +func getEndpoint() string { + const svc = "zadig-minio" + endpoint := config.S3StorageEndpoint() + newEndpoint := fmt.Sprintf("%s.%s%s", svc, config.Namespace(), strings.TrimPrefix(endpoint, svc)) + return newEndpoint +} + +func FindS3ById(id string) (*S3, error) { + storage, err := commonrepo.NewS3StorageColl().Find(id) + if err != nil { + log.Warnf("Failed to find s3 in db, err: %s", err) + return nil, err + } + + return &S3{S3Storage: storage}, nil +} + // 获取内置的s3 func FindInternalS3() *S3 { storage := &models.S3Storage{ Ak: config.S3StorageAK(), Sk: config.S3StorageSK(), - Endpoint: config.S3StorageEndpoint(), + Endpoint: getEndpoint(), Bucket: config.S3StorageBucket(), Insecure: config.S3StorageProtocol() == "http", } diff --git a/pkg/microservice/aslan/core/common/service/scmnotify/client.go b/pkg/microservice/aslan/core/common/service/scmnotify/client.go index f7299ce33307489062d20c635d2ae0d9dd3ee5c7..3eea869869cdf1f5b0ee4677e702d226a704b15c 100644 --- a/pkg/microservice/aslan/core/common/service/scmnotify/client.go +++ b/pkg/microservice/aslan/core/common/service/scmnotify/client.go @@ -17,17 +17,21 @@ limitations under the License. package scmnotify import ( + "context" "fmt" "strconv" "strings" + giteeClient "gitee.com/openeuler/go-gitee/gitee" "github.com/pkg/errors" "github.com/xanzy/go-gitlab" "go.uber.org/zap" "github.com/koderover/zadig/pkg/microservice/aslan/config" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" - "github.com/koderover/zadig/pkg/shared/codehost" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/gitee" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" "github.com/koderover/zadig/pkg/tool/gerrit" gitlabtool "github.com/koderover/zadig/pkg/tool/git/gitlab" "github.com/koderover/zadig/pkg/tool/log" @@ -55,13 +59,13 @@ func (c *Client) Comment(notify *models.Notification) error { } } - codeHostDetail, err := codehost.GetCodeHostInfoByID(notify.CodehostID) + codeHostDetail, err := systemconfig.New().GetCodeHost(notify.CodehostID) if err != nil { return errors.Wrapf(err, "codehost %d not found to comment", notify.CodehostID) } - if strings.ToLower(codeHostDetail.Type) == "gitlab" { + if strings.ToLower(codeHostDetail.Type) == setting.SourceFromGitlab { var note *gitlab.Note - cli, err := gitlabtool.NewClient(codeHostDetail.Address, codeHostDetail.AccessToken) + cli, err := gitlabtool.NewClient(codeHostDetail.ID, codeHostDetail.Address, codeHostDetail.AccessToken, config.ProxyHTTPSAddr(), codeHostDetail.EnableProxy) if err != nil { c.logger.Errorf("create gitlab client failed err: %v", err) return fmt.Errorf("create gitlab client failed err: %v", err) @@ -90,12 +94,12 @@ func (c *Client) Comment(notify *models.Notification) error { return fmt.Errorf("failed to comment gitlab due to %s/%d %v", notify.ProjectID, notify.PrID, err) } } else if strings.ToLower(codeHostDetail.Type) == gerrit.CodehostTypeGerrit { - cli := gerrit.NewClient(codeHostDetail.Address, codeHostDetail.AccessToken) + cli := gerrit.NewClient(codeHostDetail.Address, codeHostDetail.AccessToken, config.ProxyHTTPSAddr(), codeHostDetail.EnableProxy) for _, task := range notify.Tasks { // create task created comment if !task.FirstCommented && task.Status == config.TaskStatusReady { if e := cli.SetReview( - notify.ProjectID, + notify.RepoName, notify.PrID, fmt.Sprintf(""+ "%s ⏱️ %s/v1/projects/detail/%s/pipelines/multi/%s/%d", @@ -154,6 +158,35 @@ func (c *Client) Comment(notify *models.Notification) error { } } } + } else if strings.ToLower(codeHostDetail.Type) == setting.SourceFromGitee { + cli := gitee.NewClient(codeHostDetail.ID, codeHostDetail.AccessToken, config.ProxyHTTPSAddr(), codeHostDetail.EnableProxy) + var pullRequestComments giteeClient.PullRequestComments + if notify.CommentID == "" { + // create comment + pullRequestComments, err = cli.CreateMergeRequestComment(context.Background(), + notify.RepoOwner, notify.RepoName, int32(notify.PrID), giteeClient.PullRequestCommentPostParam{ + Body: comment, + }, + ) + + if err == nil { + notify.CommentID = strconv.Itoa(int(pullRequestComments.Id)) + } + } else { + // update comment + commentID, err := strconv.Atoi(notify.CommentID) + if err != nil { + return fmt.Errorf("failed to atoi commentID %v,err: %s", notify.CommentID, err) + } + err = cli.UpdateMergeRequestComment(context.Background(), + notify.RepoOwner, notify.RepoName, int32(commentID), giteeClient.PullRequestCommentPatchParam{ + Body: comment, + }) + } + + if err != nil { + return fmt.Errorf("failed to comment gitee due to %s/%d %v", notify.ProjectID, notify.PrID, err) + } } else { return fmt.Errorf("non gitlab source not supported to comment") } diff --git a/pkg/microservice/aslan/core/common/service/scmnotify/scmnotify.go b/pkg/microservice/aslan/core/common/service/scmnotify/scmnotify.go index 48bc25d3dc1ace3728f573d5e5602b44241866ff..38c98efe46b7be7a1905ae2dfc434a26e1f71e11 100644 --- a/pkg/microservice/aslan/core/common/service/scmnotify/scmnotify.go +++ b/pkg/microservice/aslan/core/common/service/scmnotify/scmnotify.go @@ -23,7 +23,6 @@ import ( "os" "strings" - "github.com/xanzy/go-gitlab" "go.uber.org/zap" "github.com/koderover/zadig/pkg/microservice/aslan/config" @@ -52,17 +51,20 @@ func NewService() *Service { } func (s *Service) SendInitWebhookComment( - mainRepo *models.MainHookRepo, prID int, baseURI string, isPipeline, isTest bool, logger *zap.SugaredLogger, + mainRepo *models.MainHookRepo, prID int, baseURI string, isPipeline, isTest, isScanning bool, logger *zap.SugaredLogger, ) (*models.Notification, error) { notification := &models.Notification{ CodehostID: mainRepo.CodehostID, PrID: prID, - ProjectID: strings.TrimLeft(mainRepo.RepoOwner+"/"+mainRepo.RepoName, "/"), + ProjectID: strings.TrimLeft(mainRepo.GetRepoNamespace()+"/"+mainRepo.RepoName, "/"), BaseURI: baseURI, IsPipeline: isPipeline, IsTest: isTest, + IsScanning: isScanning, Label: mainRepo.GetLabelValue(), Revision: mainRepo.Revision, + RepoOwner: mainRepo.RepoOwner, + RepoName: mainRepo.RepoName, } if err := s.Client.Comment(notification); err != nil { @@ -95,7 +97,7 @@ func (s *Service) SendErrWebhookComment( notification := &models.Notification{ CodehostID: mainRepo.CodehostID, PrID: prID, - ProjectID: strings.TrimLeft(mainRepo.RepoOwner+"/"+mainRepo.RepoName, "/"), + ProjectID: strings.TrimLeft(mainRepo.GetRepoNamespace()+"/"+mainRepo.RepoName, "/"), BaseURI: baseURI, IsPipeline: isPipeline, IsTest: isTest, @@ -234,81 +236,6 @@ func (s *Service) UpdateWebhookComment(task *task.Task, logger *zap.SugaredLogge return nil } -// UpdateDiffNote 调用gitlab接口更新DiffNote，并更新到数据库 -func (s *Service) UpdateDiffNote(task *task.Task, logger *zap.SugaredLogger) (err error) { - if task.WorkflowArgs.NotificationID == "" { - return - } - - var notification *models.Notification - if notification, err = s.Coll.Find(task.WorkflowArgs.NotificationID); err != nil { - logger.Errorf("can't find notification by id %s %s", task.WorkflowArgs.NotificationID, err) - return err - } - - isAllTaskSucceed := true - for _, nTask := range notification.Tasks { - if nTask.Status != config.TaskStatusFailed && nTask.Status != config.TaskStatusCancelled && - nTask.Status != config.TaskStatusPass && nTask.Status != config.TaskStatusTimeout { - // 存在任务没有执行完，直接返回 - return nil - } - // 任务都执行完了，确认是否有未成功的任务 - if nTask.Status != config.TaskStatusPass { - isAllTaskSucceed = false - break - } - } - - body := "KodeRover CI 检查通过" - if !isAllTaskSucceed { - body = "KodeRover CI 检查失败" - } - - opt := &mongodb.DiffNoteFindOpt{ - CodehostID: notification.CodehostID, - ProjectID: notification.ProjectID, - MergeRequestID: notification.PrID, - } - diffNote, err := s.DiffNoteColl.Find(opt) - if err != nil { - logger.Errorf("can't find notification by id %s %v", task.WorkflowArgs.NotificationID, err) - return err - } - - cli, _ := gitlab.NewOAuthClient(diffNote.Repo.OauthToken, gitlab.WithBaseURL(diffNote.Repo.Address)) - - // 更新note body - noteBodyOpt := &gitlab.UpdateMergeRequestDiscussionNoteOptions{ - Body: &body, - } - _, _, err = cli.Discussions.UpdateMergeRequestDiscussionNote(diffNote.Repo.ProjectID, diffNote.MergeRequestID, diffNote.DiscussionID, diffNote.NoteID, noteBodyOpt) - if err != nil { - logger.Errorf("UpdateMergeRequestDiscussionNote failed, err: %v", err) - return err - } - - // 更新resolved状态 - resolveOpt := &gitlab.UpdateMergeRequestDiscussionNoteOptions{ - Resolved: &isAllTaskSucceed, - } - _, _, err = cli.Discussions.UpdateMergeRequestDiscussionNote(diffNote.Repo.ProjectID, diffNote.MergeRequestID, diffNote.DiscussionID, diffNote.NoteID, resolveOpt) - if err != nil { - logger.Errorf("UpdateMergeRequestDiscussionNote failed, err: %v", err) - return err - } - - diffNote.Resolved = isAllTaskSucceed - diffNote.Body = body - err = s.DiffNoteColl.Update(diffNote.ObjectID.Hex(), "", diffNote.Body, diffNote.Resolved) - if err != nil { - logger.Errorf("UpdateDiscussionInfo failed, err: %v", err) - return err - } - - return nil -} - func downloadReport(taskInfo *task.Task, fileName, testName string, logger *zap.SugaredLogger) (*models.TestSuite, error) { var store *s3.S3 var err error @@ -482,6 +409,71 @@ func (s *Service) UpdateWebhookCommentForTest(task *task.Task, logger *zap.Sugar return nil } +func (s *Service) UpdateWebhookCommentForScanning(task *task.Task, logger *zap.SugaredLogger) (err error) { + if task.ScanningArgs.NotificationID == "" { + return + } + + var notification *models.Notification + if notification, err = s.Coll.Find(task.ScanningArgs.NotificationID); err != nil { + logger.Errorf("can't find notification by id %s %s", task.TestArgs.NotificationID, err) + return err + } + + var tasks []*models.NotificationTask + var taskExist bool + var shouldComment bool + + status := convertTaskStatusToNotificationTaskStatus(task.Status) + for _, nTask := range notification.Tasks { + if nTask.ID == task.TaskID { + shouldComment = nTask.Status != status + scmTask := &models.NotificationTask{ + ProductName: task.ProductName, + TestName: task.PipelineName, + ID: task.TaskID, + ScanningName: task.ScanningArgs.ScanningName, + ScanningID: task.ScanningArgs.ScanningID, + Status: status, + } + + tasks = append(tasks, scmTask) + taskExist = true + } else { + tasks = append(tasks, nTask) + } + } + + if !taskExist { + tasks = append(tasks, &models.NotificationTask{ + ProductName: task.ProductName, + TestName: task.PipelineName, + ID: task.TaskID, + ScanningName: task.ScanningArgs.ScanningName, + ScanningID: task.ScanningArgs.ScanningID, + Status: status, + }) + shouldComment = true + } + + if shouldComment { + notification.Tasks = tasks + if err = s.Client.Comment(notification); err != nil { + // cannot return error here since the upsert operation is required for further use. + logger.Warnf("failed to comment %s, %v", notification.ToString(), err) + } + + if err = s.Coll.Upsert(notification); err != nil { + logger.Warnf("can't upsert notification by id %s", notification.ID) + return + } + } else { + logger.Infof("status not changed of task %s %d, skip to update comment", task.PipelineName, task.TaskID) + } + + return nil +} + func (s *Service) UpdatePipelineWebhookComment(task *task.Task, logger *zap.SugaredLogger) (err error) { if task.TaskArgs == nil { logger.Warnf("taskArgs of %s is nil", task.PipelineName) diff --git a/pkg/microservice/aslan/core/common/service/service.go b/pkg/microservice/aslan/core/common/service/service.go index 6a7bc9fd84dd15ecd90f98472176f6f7c16587de..8272099be395ba1f965d306f7e1389f3ed9775f7 100644 --- a/pkg/microservice/aslan/core/common/service/service.go +++ b/pkg/microservice/aslan/core/common/service/service.go @@ -21,15 +21,18 @@ import ( "encoding/json" "errors" "fmt" + "net/url" "regexp" + "strconv" "strings" templ "text/template" + "time" "go.uber.org/zap" "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/yaml" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" templatemodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" @@ -37,14 +40,17 @@ import ( templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/webhook" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/codehost" - "github.com/koderover/zadig/pkg/shared/poetry" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types" "github.com/koderover/zadig/pkg/util/converter" yamlutil "github.com/koderover/zadig/pkg/util/yaml" ) +const ( + InterceptCommitID = 8 +) + type yamlPreview struct { Kind string `json:"kind"` } @@ -70,6 +76,7 @@ type ServiceTmplObject struct { EnvStatuses []*commonmodels.EnvStatus `json:"env_statuses,omitempty"` From string `json:"from,omitempty"` HealthChecks []*commonmodels.PmHealthCheck `json:"health_checks"` + EnvName string `json:"env_name"` } type ServiceProductMap struct { @@ -82,6 +89,7 @@ type ServiceProductMap struct { Visibility string `json:"visibility,omitempty"` CodehostID int `json:"codehost_id"` RepoOwner string `json:"repo_owner"` + RepoNamespace string `json:"repo_namespace"` RepoName string `json:"repo_name"` RepoUUID string `json:"repo_uuid"` BranchName string `json:"branch_name"` @@ -123,56 +131,25 @@ func ListServiceTemplate(productName string, log *zap.SugaredLogger) (*ServiceTm } for _, serviceObject := range services { - // FIXME: 兼容老数据，想办法干掉这个 - if serviceObject.Source == setting.SourceFromGitlab && serviceObject.CodehostID == 0 { - gitlabAddress, err := GetGitlabAddress(serviceObject.SrcPath) - if err != nil { - log.Errorf("无法从原有数据中恢复加载信息, GetGitlabAddr failed err: %+v", err) - return nil, e.ErrListTemplate.AddDesc(err.Error()) - } - - details, err := codehost.ListCodehostDetial() - if err != nil { - log.Errorf("无法从原有数据中恢复加载信息, listCodehostDetail failed err: %+v", err) - return nil, e.ErrListTemplate.AddDesc(err.Error()) - } - for _, detail := range details { - if strings.Contains(detail.Address, gitlabAddress) { - serviceObject.CodehostID = detail.ID - } + if serviceObject.Source == setting.SourceFromGitlab { + if serviceObject.CodehostID == 0 { + return nil, e.ErrListTemplate.AddDesc("codehost id is empty") } - _, owner, r, branch, loadPath, _, err := GetOwnerRepoBranchPath(serviceObject.SrcPath) + err = fillServiceRepoInfo(serviceObject) if err != nil { - log.Errorf("Failed to load info from url: %s, the error is: %+v", serviceObject.SrcPath, err) + log.Errorf("Failed to load info from url: %s, the error is: %s", serviceObject.SrcPath, err) return nil, e.ErrListTemplate.AddDesc(fmt.Sprintf("Failed to load info from url: %s, the error is: %+v", serviceObject.SrcPath, err)) } - // 万一codehost被删了，找不到 - if serviceObject.CodehostID == 0 { - log.Errorf("Failed to find the old code host info") - return nil, e.ErrListTemplate.AddDesc("无法找到原有的codehost信息，请确认codehost仍然存在") - } - serviceObject.RepoOwner = owner - serviceObject.RepoName = r - serviceObject.BranchName = branch - serviceObject.LoadPath = loadPath serviceObject.LoadFromDir = true } else if serviceObject.Source == setting.SourceFromGithub && serviceObject.RepoName == "" { - address, owner, r, branch, loadPath, _, err := GetOwnerRepoBranchPath(serviceObject.SrcPath) - if err != nil { - return nil, err + if serviceObject.CodehostID == 0 { + return nil, e.ErrListTemplate.AddDesc("codehost id is empty") } - - detail, err := codehost.GetCodeHostInfo( - &codehost.Option{CodeHostType: poetry.GitHubProvider, Address: address, Namespace: owner}) + err = fillServiceRepoInfo(serviceObject) if err != nil { - log.Errorf("get github codeHostInfo failed, err:%v", err) return nil, err } - serviceObject.CodehostID = detail.ID - serviceObject.RepoOwner = owner - serviceObject.RepoName = r - serviceObject.BranchName = branch - serviceObject.LoadPath = loadPath + serviceObject.LoadFromDir = true } @@ -186,6 +163,7 @@ func ListServiceTemplate(productName string, log *zap.SugaredLogger) (*ServiceTm Visibility: serviceObject.Visibility, CodehostID: serviceObject.CodehostID, RepoOwner: serviceObject.RepoOwner, + RepoNamespace: serviceObject.GetRepoNamespace(), RepoName: serviceObject.RepoName, RepoUUID: serviceObject.RepoUUID, BranchName: serviceObject.BranchName, @@ -221,6 +199,28 @@ func ListWorkloadTemplate(productName, envName string, log *zap.SugaredLogger) ( return resp, e.ErrListTemplate.AddDesc(err.Error()) } + currentServiceNames := sets.NewString() + for _, service := range services { + currentServiceNames.Insert(service.ServiceName) + } + + servicesInExternalEnv, _ := commonrepo.NewServicesInExternalEnvColl().List(&commonrepo.ServicesInExternalEnvArgs{ + ProductName: productName, + EnvName: envName, + }) + + externalServiceNames := sets.NewString() + for _, serviceInExternalEnv := range servicesInExternalEnv { + if !currentServiceNames.Has(serviceInExternalEnv.ServiceName) { + externalServiceNames.Insert(serviceInExternalEnv.ServiceName) + } + } + + if len(externalServiceNames) > 0 { + newServices, _ := commonrepo.NewServiceColl().ListExternalWorkloadsBy(productName, "", externalServiceNames.List()...) + services = append(services, newServices...) + } + for _, serviceObject := range services { spmap := &ServiceProductMap{ Service: serviceObject.ServiceName, @@ -294,57 +294,34 @@ func GetServiceTemplate(serviceName, serviceType, productName, excludeStatus str return resp, e.ErrGetTemplate.AddDesc(errMsg) } - if resp.Source == setting.SourceFromGitlab && resp.RepoName == "" { - if gitlabAddress, err := GetGitlabAddress(resp.SrcPath); err == nil { - if details, err := codehost.ListCodehostDetial(); err == nil { - for _, detail := range details { - if strings.Contains(detail.Address, gitlabAddress) { - resp.GerritCodeHostID = detail.ID - resp.CodehostID = detail.ID - } - } - _, owner, r, branch, loadPath, pathType, err := GetOwnerRepoBranchPath(resp.SrcPath) - if err != nil { - log.Errorf("Failed to load info from url: %s, the error is: %+v", resp.SrcPath, err) - return nil, e.ErrGetService.AddDesc(fmt.Sprintf("Failed to load info from url: %s, the error is: %+v", resp.SrcPath, err)) - } - // 万一codehost被删了，找不到 - if resp.CodehostID == 0 { - log.Errorf("Failed to find the old code host info") - return nil, e.ErrListTemplate.AddDesc("无法找到原有的codehost信息，请确认codehost仍然存在") - } - resp.RepoOwner = owner - resp.RepoName = r - resp.BranchName = branch - resp.LoadPath = loadPath - resp.LoadFromDir = pathType == "tree" - return resp, nil - } - errMsg := fmt.Sprintf("[ServiceTmpl.Find] ListCodehostDetail %s error: %v", serviceName, err) - log.Error(errMsg) - } else { - errMsg := fmt.Sprintf("[ServiceTmpl.Find] GetGitlabAddress %s error: %v", serviceName, err) + if resp.Type == setting.PMDeployType { + err = fillPmInfo(resp) + if err != nil { + errMsg := fmt.Sprintf("[ServiceTmpl.Find] %s fillPmInfo error: %v", serviceName, err) log.Error(errMsg) + return resp, e.ErrGetTemplate.AddDesc(errMsg) } + } - } else if resp.Source == setting.SourceFromGithub && resp.GerritCodeHostID == 0 { - address, owner, r, branch, loadPath, _, err := GetOwnerRepoBranchPath(resp.SrcPath) + if resp.Source == setting.SourceFromGitlab && resp.RepoName == "" { + if resp.CodehostID == 0 { + return nil, e.ErrGetTemplate.AddDesc("Please confirm if codehost exists") + } + err = fillServiceRepoInfo(resp) if err != nil { - return nil, err + log.Errorf("Failed to load info from url: %s, the error is: %s", resp.SrcPath, err) + return nil, e.ErrGetService.AddDesc(fmt.Sprintf("Failed to load info from url: %s, the error is: %+v", resp.SrcPath, err)) } - - detail, err := codehost.GetCodeHostInfo( - &codehost.Option{CodeHostType: poetry.GitHubProvider, Address: address, Namespace: owner}) + return resp, nil + } else if resp.Source == setting.SourceFromGithub { + if resp.CodehostID == 0 { + return nil, e.ErrGetTemplate.AddDesc("Please confirm if codehost exists") + } + err = fillServiceRepoInfo(resp) if err != nil { - log.Errorf("get github codeHostInfo failed, err:%v", err) return nil, err } - resp.CodehostID = detail.ID - resp.RepoOwner = owner - resp.RepoName = r - resp.BranchName = branch - resp.LoadPath = loadPath - resp.LoadFromDir = true + return resp, nil } else if resp.Source == setting.SourceFromGUI { @@ -384,10 +361,39 @@ func GetServiceTemplate(serviceName, serviceType, productName, excludeStatus str } } } - + resp.RepoNamespace = resp.GetRepoNamespace() return resp, nil } +func fillPmInfo(svc *commonmodels.Service) error { + pms, err := commonrepo.NewPrivateKeyColl().List(&commonrepo.PrivateKeyArgs{}) + if err != nil { + if commonrepo.IsErrNoDocuments(err) { + return nil + } + return err + } + pmMaps := make(map[string]*commonmodels.PrivateKey, len(pms)) + for _, pm := range pms { + pmMaps[pm.ID.Hex()] = pm + } + for _, envStatus := range svc.EnvStatuses { + if pm, ok := pmMaps[envStatus.HostID]; ok { + envStatus.PmInfo = &commonmodels.PmInfo{ + ID: pm.ID, + Name: pm.Name, + IP: pm.IP, + Port: pm.Port, + Status: pm.Status, + Label: pm.Label, + IsProd: pm.IsProd, + Provider: pm.Provider, + } + } + } + return nil +} + func UpdatePmServiceTemplate(username string, args *ServiceTmplBuildObject, log *zap.SugaredLogger) error { //该请求来自环境中的服务更新时，from=createEnv if args.ServiceTmplObject.From == "" { @@ -411,10 +417,11 @@ func UpdatePmServiceTemplate(username string, args *ServiceTmplBuildObject, log return err } preService.HealthChecks = args.ServiceTmplObject.HealthChecks - preService.EnvConfigs = args.ServiceTmplObject.EnvConfigs preService.Revision = rev preService.CreateBy = username preService.BuildName = args.Build.Name + preService.EnvConfigs = args.ServiceTmplObject.EnvConfigs + preService.EnvStatuses = args.ServiceTmplObject.EnvStatuses if err := commonrepo.NewServiceColl().Delete(preService.ServiceName, setting.PMDeployType, args.ServiceTmplObject.ProductName, setting.ProductStatusDeleting, preService.Revision); err != nil { return err @@ -460,7 +467,6 @@ func UpdatePmServiceTemplate(username string, args *ServiceTmplBuildObject, log return e.ErrUpdateService.AddDesc("update current build failed") } } - } if err := commonrepo.NewServiceColl().Create(preService); err != nil { @@ -478,34 +484,61 @@ func DeleteServiceWebhookByName(serviceName, productName string, logger *zap.Sug ProcessServiceWebhook(nil, svc, serviceName, logger) } -func ProcessServiceWebhook(updated, current *commonmodels.Service, serviceName string, logger *zap.SugaredLogger) { - // helm service doesn't support webhook - if current.Type == setting.HelmDeployType { - return +func needProcessWebhook(source string) bool { + if source == setting.ServiceSourceTemplate || source == setting.SourceFromZadig || source == setting.SourceFromGerrit || + source == "" || source == setting.SourceFromExternal || source == setting.SourceFromChartTemplate || source == setting.SourceFromChartRepo { + return false } + return true +} + +func ProcessServiceWebhook(updated, current *commonmodels.Service, serviceName string, logger *zap.SugaredLogger) { var action string var updatedHooks, currentHooks []*webhook.WebHook if updated != nil { - if updated.Source == setting.SourceFromZadig || updated.Source == setting.SourceFromGerrit || updated.Source == "" || updated.Source == setting.SourceFromExternal { + if !needProcessWebhook(updated.Source) { return } action = "add" - address := getAddressFromPath(updated.SrcPath, updated.RepoOwner, updated.RepoName, logger.Desugar()) + address, err := GetGitlabAddress(updated.SrcPath) + if err != nil { + log.Errorf("failed to parse codehost address, err: %s", err) + return + } if address == "" { return } - updatedHooks = append(updatedHooks, &webhook.WebHook{Owner: updated.RepoOwner, Repo: updated.RepoName, Address: address, Name: "trigger", CodeHostID: updated.CodehostID}) + updatedHooks = append(updatedHooks, &webhook.WebHook{ + Owner: updated.RepoOwner, + Namespace: updated.GetRepoNamespace(), + Repo: updated.RepoName, + Address: address, + Name: "trigger", + CodeHostID: updated.CodehostID, + }) } if current != nil { - if current.Source == setting.SourceFromZadig || current.Source == setting.SourceFromGerrit || current.Source == "" || current.Source == setting.SourceFromExternal { + if !needProcessWebhook(current.Source) { return } action = "remove" - address := getAddressFromPath(current.SrcPath, current.RepoOwner, current.RepoName, logger.Desugar()) + address, err := GetGitlabAddress(current.SrcPath) + if err != nil { + log.Errorf("failed to parse codehost address, err: %s", err) + return + } + //address := getAddressFromPath(current.SrcPath, current.GetRepoNamespace(), current.RepoName, logger.Desugar()) if address == "" { return } - currentHooks = append(currentHooks, &webhook.WebHook{Owner: current.RepoOwner, Repo: current.RepoName, Address: address, Name: "trigger", CodeHostID: current.CodehostID}) + currentHooks = append(currentHooks, &webhook.WebHook{ + Owner: current.RepoOwner, + Namespace: current.GetRepoNamespace(), + Repo: current.RepoName, + Address: address, + Name: "trigger", + CodeHostID: current.CodehostID, + }) } if updated != nil && current != nil { action = "update" @@ -593,7 +626,67 @@ func ExtractImageName(imageURI string) string { return "" } -func parseImagesByPattern(nested map[string]interface{}, patterns []map[string]string) ([]*models.Container, error) { +// ExtractImageRegistry extract registry url from total image uri +func ExtractImageRegistry(imageURI string) (string, error) { + subMatchAll := imageParseRegex.FindStringSubmatch(imageURI) + exNames := imageParseRegex.SubexpNames() + for i, matchedStr := range subMatchAll { + if i != 0 && matchedStr != "" && matchedStr != ":" { + if exNames[i] == "repo" { + u, err := url.Parse(matchedStr) + if err != nil { + return "", err + } + if len(u.Scheme) > 0 { + matchedStr = strings.TrimPrefix(matchedStr, fmt.Sprintf("%s://", u.Scheme)) + } + return matchedStr, nil + } + } + } + return "", fmt.Errorf("failed to extract registry url") +} + +// ExtractImageTag extract image tag from total image uri +func ExtractImageTag(imageURI string) string { + subMatchAll := imageParseRegex.FindStringSubmatch(imageURI) + exNames := imageParseRegex.SubexpNames() + for i, matchedStr := range subMatchAll { + if i != 0 && matchedStr != "" && matchedStr != ":" { + if exNames[i] == "tag" { + return matchedStr + } + } + } + return "" +} + +// ExtractRegistryNamespace extract registry namespace from image uri +func ExtractRegistryNamespace(imageURI string) string { + imageURI = strings.TrimPrefix(imageURI, "http://") + imageURI = strings.TrimPrefix(imageURI, "https://") + + imageComponent := strings.Split(imageURI, "/") + if len(imageComponent) <= 2 { + return "" + } + + nsComponent := imageComponent[1 : len(imageComponent)-1] + return strings.Join(nsComponent, "/") +} + +func generateUniquePath(pathData map[string]string) string { + keys := []string{setting.PathSearchComponentRepo, setting.PathSearchComponentImage, setting.PathSearchComponentTag} + values := make([]string, 0) + for _, key := range keys { + if value := pathData[key]; value != "" { + values = append(values, value) + } + } + return strings.Join(values, "-") +} + +func parseImagesByPattern(nested map[string]interface{}, patterns []map[string]string) ([]*commonmodels.Container, error) { flatMap, err := converter.Flatten(nested) if err != nil { return nil, err @@ -602,16 +695,24 @@ func parseImagesByPattern(nested map[string]interface{}, patterns []map[string]s if err != nil { return nil, err } - ret := make([]*models.Container, 0) + ret := make([]*commonmodels.Container, 0) + usedImagePath := sets.NewString() for _, searchResult := range matchedPath { + uniquePath := generateUniquePath(searchResult) + if usedImagePath.Has(uniquePath) { + continue + } + usedImagePath.Insert(uniquePath) imageUrl, err := GeneImageURI(searchResult, flatMap) if err != nil { return nil, err } - ret = append(ret, &models.Container{ - Name: ExtractImageName(imageUrl), - Image: imageUrl, - ImagePath: &models.ImagePathSpec{ + name := ExtractImageName(imageUrl) + ret = append(ret, &commonmodels.Container{ + Name: name, + ImageName: name, + Image: imageUrl, + ImagePath: &commonmodels.ImagePathSpec{ Repo: searchResult[setting.PathSearchComponentRepo], Image: searchResult[setting.PathSearchComponentImage], Tag: searchResult[setting.PathSearchComponentTag], @@ -621,7 +722,7 @@ func parseImagesByPattern(nested map[string]interface{}, patterns []map[string]s return ret, nil } -func ParseImagesByRules(nested map[string]interface{}, matchRules []*template.ImageSearchingRule) ([]*models.Container, error) { +func ParseImagesByRules(nested map[string]interface{}, matchRules []*templatemodels.ImageSearchingRule) ([]*commonmodels.Container, error) { patterns := make([]map[string]string, 0) for _, rule := range matchRules { if !rule.InUse { @@ -654,7 +755,7 @@ func getServiceParsePatterns(productName string) ([]map[string]string, error) { } // ParseImagesForProductService for product service -func ParseImagesForProductService(nested map[string]interface{}, serviceName, productName string) ([]*models.Container, error) { +func ParseImagesForProductService(nested map[string]interface{}, serviceName, productName string) ([]*commonmodels.Container, error) { patterns, err := getServiceParsePatterns(productName) if err != nil { log.Errorf("failed to get image parse patterns for service %s in project %s, err: %s", serviceName, productName, err) @@ -668,10 +769,10 @@ func ParseImagesByPresetRules(flatMap map[string]interface{}) ([]map[string]stri return yamlutil.SearchByPattern(flatMap, presetPatterns) } -func GetPresetRules() []*template.ImageSearchingRule { - ret := make([]*template.ImageSearchingRule, 0, len(presetPatterns)) +func GetPresetRules() []*templatemodels.ImageSearchingRule { + ret := make([]*templatemodels.ImageSearchingRule, 0, len(presetPatterns)) for id, pattern := range presetPatterns { - ret = append(ret, &template.ImageSearchingRule{ + ret = append(ret, &templatemodels.ImageSearchingRule{ Repo: pattern[setting.PathSearchComponentRepo], Image: pattern[setting.PathSearchComponentImage], Tag: pattern[setting.PathSearchComponentTag], @@ -684,6 +785,7 @@ func GetPresetRules() []*template.ImageSearchingRule { type Variable struct { SERVICE string + IMAGE_NAME string TIMESTAMP string TASK_ID string REPO_COMMIT_ID string @@ -694,6 +796,130 @@ type Variable struct { REPO_PR string } +type candidate struct { + Branch string + Tag string + CommitID string + PR int + TaskID int64 + Timestamp string + ProductName string + ServiceName string + ImageName string + EnvName string +} + +// releaseCandidate 根据 TaskID 生成编译镜像Tag或者二进制包后缀 +// TODO: max length of a tag is 128 +func ReleaseCandidate(builds []*types.Repository, taskID int64, productName, serviceName, envName, imageName, deliveryType string) string { + timeStamp := time.Now().Format("20060102150405") + + if imageName == "" { + imageName = serviceName + } + if len(builds) == 0 { + switch deliveryType { + case config.TarResourceType: + return fmt.Sprintf("%s-%s", serviceName, timeStamp) + default: + return fmt.Sprintf("%s:%s", imageName, timeStamp) + } + } + + first := builds[0] + for index, build := range builds { + if build.IsPrimary { + first = builds[index] + } + } + + // 替换 Tag 和 Branch 中的非法字符为 "-", 避免 docker build 失败 + var ( + reg = regexp.MustCompile(`[^\w.-]`) + customImageRule *template.CustomRule + customTarRule *template.CustomRule + commitID = first.CommitID + ) + + if project, err := templaterepo.NewProductColl().Find(productName); err != nil { + log.Errorf("find project err:%s", err) + } else { + customImageRule = project.CustomImageRule + customTarRule = project.CustomTarRule + } + + if len(commitID) > InterceptCommitID { + commitID = commitID[0:InterceptCommitID] + } + + candidate := &candidate{ + Branch: string(reg.ReplaceAll([]byte(first.Branch), []byte("-"))), + CommitID: commitID, + PR: first.PR, + Tag: string(reg.ReplaceAll([]byte(first.Tag), []byte("-"))), + EnvName: envName, + Timestamp: timeStamp, + TaskID: taskID, + ProductName: productName, + ServiceName: serviceName, + ImageName: imageName, + } + switch deliveryType { + case config.TarResourceType: + newTarRule := replaceVariable(customTarRule, candidate) + if strings.Contains(newTarRule, ":") { + return strings.Replace(newTarRule, ":", "-", -1) + } + return newTarRule + default: + return replaceVariable(customImageRule, candidate) + } +} + +// There are four situations in total +// 1.Execute workflow selection tag build +// 2.Execute workflow selection branch and pr build +// 3.Execute workflow selection branch pr build +// 4.Execute workflow selection branch build +func replaceVariable(customRule *template.CustomRule, candidate *candidate) string { + var currentRule string + if candidate.Tag != "" { + if customRule == nil { + return fmt.Sprintf("%s:%s-%s", candidate.ServiceName, candidate.Timestamp, candidate.Tag) + } + currentRule = customRule.TagRule + } else if candidate.Branch != "" && candidate.PR != 0 { + if customRule == nil { + return fmt.Sprintf("%s:%s-%d-%s-pr-%d", candidate.ServiceName, candidate.Timestamp, candidate.TaskID, candidate.Branch, candidate.PR) + } + currentRule = customRule.PRAndBranchRule + } else if candidate.Branch == "" && candidate.PR != 0 { + if customRule == nil { + return fmt.Sprintf("%s:%s-%d-pr-%d", candidate.ServiceName, candidate.Timestamp, candidate.TaskID, candidate.PR) + } + currentRule = customRule.PRRule + } else if candidate.Branch != "" && candidate.PR == 0 { + if customRule == nil { + return fmt.Sprintf("%s:%s-%d-%s", candidate.ServiceName, candidate.Timestamp, candidate.TaskID, candidate.Branch) + } + currentRule = customRule.BranchRule + } + + currentRule = ReplaceRuleVariable(currentRule, &Variable{ + SERVICE: candidate.ServiceName, + IMAGE_NAME: candidate.ImageName, + TIMESTAMP: candidate.Timestamp, + TASK_ID: strconv.FormatInt(candidate.TaskID, 10), + REPO_COMMIT_ID: candidate.CommitID, + PROJECT: candidate.ProductName, + ENV_NAME: candidate.EnvName, + REPO_TAG: candidate.Tag, + REPO_BRANCH: candidate.Branch, + REPO_PR: strconv.Itoa(candidate.PR), + }) + return currentRule +} + func ReplaceRuleVariable(rule string, replaceValue *Variable) string { template, err := templ.New("replaceRuleVariable").Parse(rule) if err != nil { diff --git a/pkg/microservice/aslan/core/common/service/sets.go b/pkg/microservice/aslan/core/common/service/sets.go index d636068169bfb304c4d3a160d8c4ed015f5d9c28..719ae234f38cc40a5e2aaafa46f84d9a657c4eef 100644 --- a/pkg/microservice/aslan/core/common/service/sets.go +++ b/pkg/microservice/aslan/core/common/service/sets.go @@ -17,7 +17,7 @@ limitations under the License. package service type hookUniqueID struct { - owner, repo, source, name string + owner, namespace, repo, source, name string } type hookItem struct { diff --git a/pkg/microservice/aslan/core/common/service/task.go b/pkg/microservice/aslan/core/common/service/task.go index 17bd62163e80578289d6b0ee7cc8f0dfc1bf2c9d..aa203da5f72eff5235731ebac168df47537ffa0a 100644 --- a/pkg/microservice/aslan/core/common/service/task.go +++ b/pkg/microservice/aslan/core/common/service/task.go @@ -25,6 +25,7 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/nsq" @@ -40,6 +41,14 @@ type cancelMessage struct { ReqID string `json:"req_id"` } +func CancelWorkflowTaskV3(userName, pipelineName string, taskID int64, typeString config.PipelineType, requestID string, log *zap.SugaredLogger) error { + if err := CancelTask(userName, pipelineName, taskID, typeString, requestID, log); err != nil { + log.Errorf("[%s] cancel pipeline [%s:%d] error: %s", userName, pipelineName, taskID, err) + return e.ErrCancelTask.AddDesc(err.Error()) + } + return nil +} + func CancelTaskV2(userName, pipelineName string, taskID int64, typeString config.PipelineType, requestID string, log *zap.SugaredLogger) error { if err := CancelTask(userName, pipelineName, taskID, typeString, requestID, log); err != nil { @@ -74,11 +83,16 @@ func CancelTask(userName, pipelineName string, taskID int64, typeString config.P if typeString == config.WorkflowType { _ = scmNotifyService.UpdateWebhookComment(t, log) - _ = scmNotifyService.UpdateDiffNote(t, log) } else if typeString == config.TestType { _ = scmNotifyService.UpdateWebhookCommentForTest(t, log) } else if typeString == config.SingleType { _ = scmNotifyService.UpdatePipelineWebhookComment(t, log) + } else if typeString == config.ScanningType { + err = scmNotifyService.UpdateWebhookCommentForScanning(t, log) + // this logic will not affect the result of cancellation, so we only print the error in logs. + if err != nil { + log.Errorf("Failed to update webhook comment for scanning in upon cancel, the error is: %s", err) + } } return nil @@ -122,11 +136,16 @@ func CancelTask(userName, pipelineName string, taskID int64, typeString config.P if typeString == config.WorkflowType { _ = scmNotifyService.UpdateWebhookComment(t, log) - _ = scmNotifyService.UpdateDiffNote(t, log) } else if typeString == config.TestType { _ = scmNotifyService.UpdateWebhookCommentForTest(t, log) } else if typeString == config.SingleType { _ = scmNotifyService.UpdatePipelineWebhookComment(t, log) + } else if typeString == config.ScanningType { + err = scmNotifyService.UpdateWebhookCommentForScanning(t, log) + // this logic will not affect the result of cancellation, so we only print the error in logs. + if err != nil { + log.Errorf("Failed to update webhook comment for scanning in upon cancel, the error is: %s", err) + } } return nil @@ -163,3 +182,10 @@ func covertTaskToQueue(t *task.Task) *models.Queue { CreateTime: t.CreateTime, } } + +func GetWorkflowTaskCallback(taskID int64, pipelineName string) (*commonmodels.CallbackRequest, error) { + return mongodb.NewCallbackRequestColl().Find(&mongodb.CallbackFindOption{ + TaskID: taskID, + PipelineName: pipelineName, + }) +} diff --git a/pkg/microservice/aslan/core/common/service/template/dockerfile.go b/pkg/microservice/aslan/core/common/service/template/dockerfile.go new file mode 100644 index 0000000000000000000000000000000000000000..b246736b98a00941aac4206c85281fcef221848d --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/template/dockerfile.go @@ -0,0 +1,99 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package template + +import ( + "errors" + "fmt" + "strings" + + "go.uber.org/zap" + + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/setting" + dockerfileinstructions "github.com/moby/buildkit/frontend/dockerfile/instructions" + dockerfileparser "github.com/moby/buildkit/frontend/dockerfile/parser" +) + +func GetDockerfileTemplateDetail(id string, logger *zap.SugaredLogger) (*DockerfileDetail, error) { + resp := new(DockerfileDetail) + dockerfileTemplate, err := commonrepo.NewDockerfileTemplateColl().GetById(id) + if err != nil { + logger.Errorf("Failed to get dockerfile template from id: %s, the error is: %s", id, err) + return nil, err + } + variables, err := getVariables(dockerfileTemplate.Content, logger) + if err != nil { + return nil, errors.New("failed to get variables from dockerfile") + } + resp.ID = dockerfileTemplate.ID.Hex() + resp.Name = dockerfileTemplate.Name + resp.Content = dockerfileTemplate.Content + resp.Variables = variables + return resp, nil +} + +func getVariables(s string, logger *zap.SugaredLogger) ([]*commonmodels.ChartVariable, error) { + ret := make([]*commonmodels.ChartVariable, 0) + reader := strings.NewReader(s) + result, err := dockerfileparser.Parse(reader) + if err != nil { + logger.Errorf("Failed to parse the dockerfile from source, the error is: %s", err) + return []*commonmodels.ChartVariable{}, err + } + stages, metaArgs, err := dockerfileinstructions.Parse(result.AST) + if err != nil { + logger.Errorf("Failed to parse stages from generated dockerfile AST, the error is: %s", err) + return []*commonmodels.ChartVariable{}, err + } + keyMap := make(map[string]int) + for _, metaArg := range metaArgs { + for _, arg := range metaArg.Args { + if keyMap[arg.Key] == 0 { + ret = append(ret, &commonmodels.ChartVariable{ + Key: arg.Key, + Value: arg.ValueString(), + }) + keyMap[arg.Key] = 1 + } + } + } + for _, stage := range stages { + for _, command := range stage.Commands { + if command.Name() == setting.DockerfileCmdArg { + fullCommand := fmt.Sprintf("%s", command) + commandContent := strings.Split(fullCommand, " ")[1] + kv := strings.Split(commandContent, "=") + key := kv[0] + value := "" + if len(kv) > 1 { + value = kv[1] + } + // if key has not been added yet + if keyMap[key] == 0 { + ret = append(ret, &commonmodels.ChartVariable{ + Key: key, + Value: value, + }) + keyMap[key] = 1 + } + } + } + } + return ret, nil +} diff --git a/pkg/microservice/aslan/core/common/service/template/types.go b/pkg/microservice/aslan/core/common/service/template/types.go new file mode 100644 index 0000000000000000000000000000000000000000..d9dc9a4e4625f1df1e34745bcd16c87222741b0c --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/template/types.go @@ -0,0 +1,80 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package template + +import ( + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/fs" +) + +type Chart struct { + Name string `json:"name"` + CodehostID int `json:"codehostID"` + Owner string `json:"owner"` + Namespace string `json:"namespace"` + Repo string `json:"repo"` + Branch string `json:"branch"` + Path string `json:"path"` + Variables []*models.ChartVariable `json:"variables,omitempty"` + + Files []*fs.FileInfo `json:"files,omitempty"` +} + +type DockerfileTemplate struct { + Name string `json:"name"` + Content string `json:"content"` +} + +type DockerfileListObject struct { + ID string `json:"id"` + Name string `json:"name"` +} + +type DockerfileDetail struct { + ID string `json:"id"` + Name string `json:"name"` + Content string `json:"content"` + Variables []*models.ChartVariable `json:"variable"` +} + +type BuildReference struct { + BuildName string `json:"build_name"` + ProjectName string `json:"project_name"` +} + +type YamlTemplate struct { + Name string `json:"name"` + Content string `json:"content"` + Variable []*models.ChartVariable `json:"variable"` +} + +type YamlListObject struct { + ID string `json:"id"` + Name string `json:"name"` +} + +type YamlDetail struct { + ID string `json:"id"` + Name string `json:"name"` + Content string `json:"content"` + Variables []*models.ChartVariable `json:"variable"` +} + +type ServiceReference struct { + ProjectName string `json:"project_name"` + ServiceName string `json:"service_name"` +} diff --git a/pkg/microservice/aslan/core/common/service/template_product.go b/pkg/microservice/aslan/core/common/service/template_product.go index 94dd0dc5919215298ec839f7cab70cc2430de862..5546a6510a3bb0e2fabb8abfee1e465cf672e4a5 100644 --- a/pkg/microservice/aslan/core/common/service/template_product.go +++ b/pkg/microservice/aslan/core/common/service/template_product.go @@ -18,20 +18,20 @@ package service import ( "fmt" - "strings" "sync" "go.uber.org/zap" + "k8s.io/apimachinery/pkg/util/sets" + configbase "github.com/koderover/zadig/pkg/config" "github.com/koderover/zadig/pkg/microservice/aslan/config" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/collie" - "github.com/koderover/zadig/pkg/microservice/aslan/internal/cache" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/poetry" + configclient "github.com/koderover/zadig/pkg/shared/config" e "github.com/koderover/zadig/pkg/tool/errors" ) @@ -52,15 +52,11 @@ func GetProductTemplate(productName string, log *zap.SugaredLogger) (*template.P } totalFreeStyles := make([]*collie.CiPipelineResource, 0) - features, err := GetFeatures(log) - if err != nil { - log.Errorf("GetProductTemplate GetFeatures err : %v", err) - } - - if strings.Contains(features, string(config.FreestyleType)) { + cl := configclient.New(configbase.ConfigServiceAddress()) + if enable, err := cl.CheckFeature(setting.ModernWorkflowType); err == nil && enable { // CI场景onboarding流程处于第二步时，需要返回ci工作流id，用于前端跳转 collieAPIAddress := config.CollieAPIAddress() - cl := collie.New(collieAPIAddress, config.PoetryAPIRootKey()) + cl := collie.New(collieAPIAddress) if resp.ProductFeature != nil && resp.ProductFeature.DevelopHabit == "yaml" && resp.OnboardingStatus == setting.OnboardingStatusSecond && collieAPIAddress != "" { ciPipelines, err := cl.ListCIPipelines(productName, log) if err != nil { @@ -84,6 +80,13 @@ func GetProductTemplate(productName string, log *zap.SugaredLogger) (*template.P if err != nil { return resp, fmt.Errorf("ListExternalWorkloadsBy err : %s", err) } + serviceNamesSet := sets.NewString() + for _, service := range totalServices { + serviceNamesSet.Insert(service.ServiceName) + } + if len(resp.Services) > 0 { + resp.Services[0] = serviceNamesSet.List() + } } else { totalServices, err = commonrepo.NewServiceColl().ListMaxRevisionsByProduct(productName) if err != nil { @@ -101,12 +104,12 @@ func GetProductTemplate(productName string, log *zap.SugaredLogger) (*template.P return resp, fmt.Errorf("Testing.List err : %v", err) } - totalEnvs, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{Name: productName, IsSort: true}) + totalEnvs, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{Name: productName, IsSortByUpdateTime: true}) if err != nil { return resp, fmt.Errorf("Product.List err : %v", err) } - totalWorkflows, err := commonrepo.NewWorkflowColl().List(&commonrepo.ListWorkflowOption{ProductName: productName, IsSort: true}) + totalWorkflows, err := commonrepo.NewWorkflowColl().List(&commonrepo.ListWorkflowOption{Projects: []string{productName}, IsSort: true}) if err != nil { return resp, fmt.Errorf("Workflow.List err : %v", err) } @@ -116,9 +119,9 @@ func GetProductTemplate(productName string, log *zap.SugaredLogger) (*template.P return resp, fmt.Errorf("Pipeline.List err : %v", err) } - if strings.Contains(features, string(config.FreestyleType)) { + if enable, err := cl.CheckFeature(setting.ModernWorkflowType); err == nil && enable { collieAPIAddress := config.CollieAPIAddress() - cl := collie.New(collieAPIAddress, config.PoetryAPIRootKey()) + cl := collie.New(collieAPIAddress) totalFreeStyles, err = cl.ListCIPipelines(productName, log) if err != nil { log.Errorf("GetProductTemplate freestyle.List err : %v", err) @@ -134,10 +137,11 @@ func GetProductTemplate(productName string, log *zap.SugaredLogger) (*template.P if len(totalServices) > 0 { serviceObj, err := GetServiceTemplate(totalServices[0].ServiceName, totalServices[0].Type, productName, setting.ProductStatusDeleting, totalServices[0].Revision, log) if err != nil { - return resp, fmt.Errorf("GetServiceTemplate err : %v", err) + log.Errorf("GetServiceTemplate err : %s", err) + } else { + resp.LatestServiceUpdateTime = serviceObj.CreateTime + resp.LatestServiceUpdateBy = serviceObj.CreateBy } - resp.LatestServiceUpdateTime = serviceObj.CreateTime - resp.LatestServiceUpdateBy = serviceObj.CreateBy } resp.TotalBuildNum = len(totalBuilds) if len(totalBuilds) > 0 { @@ -165,26 +169,6 @@ func GetProductTemplate(productName string, log *zap.SugaredLogger) (*template.P return resp, nil } -func GetFeatures(log *zap.SugaredLogger) (string, error) { - featuresByteKey := []byte("features") - featuresByteValue, err := cache.Get(featuresByteKey) - if err != nil { - poetryCtl := poetry.New(config.PoetryAPIServer(), config.PoetryAPIRootKey()) - fs, err := poetryCtl.ListFeatures() - if err != nil { - return "", err - } - cacheValue := strings.Join(fs, ",") - // 一天过期 - if err = cache.Set(featuresByteKey, []byte(cacheValue), 86400); err != nil { - log.Errorf("getFeatures set cache err:%v", err) - } - return cacheValue, nil - } - - return string(featuresByteValue), nil -} - func FillProductTemplateVars(productTemplates []*template.Product, log *zap.SugaredLogger) error { var ( wg sync.WaitGroup @@ -204,7 +188,7 @@ func FillProductTemplateVars(productTemplates []*template.Product, log *zap.Suga <-ch wg.Done() }() - renderSet, err := GetRenderSet(tmpl.ProductName, 0, log) + renderSet, err := GetRenderSet(tmpl.ProductName, 0, true, "", log) if err != nil { errStr += fmt.Sprintf("Failed to find render set for product template, productName:%s, err:%v\n", tmpl.ProductName, err) log.Errorf("Failed to find render set for product template %s", tmpl.ProductName) diff --git a/pkg/microservice/aslan/core/templatestore/service/types.go b/pkg/microservice/aslan/core/common/service/template_store.go similarity index 69% rename from pkg/microservice/aslan/core/templatestore/service/types.go rename to pkg/microservice/aslan/core/common/service/template_store.go index da8cdeb74d737a1efa332119d61375f3e2b7b9d4..68d5269c7e4d0e277a50ffaa61ed6287858a97f5 100644 --- a/pkg/microservice/aslan/core/templatestore/service/types.go +++ b/pkg/microservice/aslan/core/common/service/template_store.go @@ -16,15 +16,12 @@ limitations under the License. package service -import "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/fs" +import "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" -type Chart struct { - Name string `json:"name"` - CodehostID int `json:"codehostID"` - Owner string `json:"owner"` - Repo string `json:"repo"` - Branch string `json:"branch"` - Path string `json:"path"` - - Files []*fs.FileInfo `json:"files,omitempty"` +func GetDockerfileTemplateContent(id string) (string, error) { + dockerfileTemplate, err := mongodb.NewDockerfileTemplateColl().GetById(id) + if err != nil { + return "", err + } + return dockerfileTemplate.Content, nil } diff --git a/pkg/microservice/aslan/core/common/service/testing.go b/pkg/microservice/aslan/core/common/service/testing.go index 5f2dc5df2ab260b5880f3006ae86129bc35d1fe5..98dfe7d1b8c759a5508cea6ca57f7e72b8263c4a 100644 --- a/pkg/microservice/aslan/core/common/service/testing.go +++ b/pkg/microservice/aslan/core/common/service/testing.go @@ -23,6 +23,8 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/webhook" "github.com/koderover/zadig/pkg/setting" e "github.com/koderover/zadig/pkg/tool/errors" ) @@ -52,7 +54,19 @@ func Delete(name, productName string, log *zap.SugaredLogger) error { return e.ErrDeleteTestModule.AddDesc("empty Name") } - err := mongodb.NewTestingColl().Delete(name, productName) + // find test data to delete webhook + testModule, err := commonrepo.NewTestingColl().Find(name, productName) + if err != nil { + return e.ErrDeleteTestModule.AddDesc(err.Error()) + } + + // delete webhooks of test + err = ProcessWebhook(nil, testModule.HookCtl.Items, webhook.TestingPrefix+name, log) + if err != nil { + return e.ErrDeleteTestModule.AddErr(err) + } + + err = mongodb.NewTestingColl().Delete(name, productName) if err != nil { log.Errorf("[Testing.Delete] %s error: %v", name, err) return e.ErrDeleteTestModule.AddErr(err) diff --git a/pkg/microservice/aslan/core/common/service/utils.go b/pkg/microservice/aslan/core/common/service/utils.go index 742826e89a1ee8a391aac63355bb5b363917ce7d..ed2ef03e753ded7ce599c56e5012bad6e4602d45 100644 --- a/pkg/microservice/aslan/core/common/service/utils.go +++ b/pkg/microservice/aslan/core/common/service/utils.go @@ -28,9 +28,7 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/notify" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/poetry" e "github.com/koderover/zadig/pkg/tool/errors" - "github.com/koderover/zadig/pkg/types/permission" ) func SendMessage(sender, title, content, requestID string, log *zap.SugaredLogger) { @@ -54,11 +52,6 @@ func SendMessage(sender, title, content, requestID string, log *zap.SugaredLogge func SendFailedTaskMessage(username, productName, name, requestID string, workflowType config.PipelineType, err error, log *zap.SugaredLogger) { title := "创建工作流任务失败" - perm := permission.WorkflowUpdateUUID - if workflowType == config.TestType { - title = "创建测试任务失败" - perm = permission.TestManageUUID - } errStr := err.Error() _, messageMap := e.ErrorMessage(err) @@ -74,13 +67,6 @@ func SendFailedTaskMessage(username, productName, name, requestID string, workfl SendMessage(username, title, content, requestID, log) return } - - // 如果是timer创建的任务，通知需要发送给该项目下有编辑工作流权限的用户 - poetryClient := poetry.New(config.PoetryAPIServer(), config.PoetryAPIRootKey()) - users, _ := poetryClient.ListProductPermissionUsers(productName, perm, log) - for _, user := range users { - SendMessage(user, title, content, requestID, log) - } } func SendErrorMessage(sender, title, requestID string, err error, log *zap.SugaredLogger) { @@ -99,10 +85,10 @@ func GetGitlabAddress(URL string) (string, error) { return fmt.Sprintf("%s://%s", uri.Scheme, uri.Host), nil } -// GetOwnerRepoBranchPath 获取gitlab路径中的owner、repo、branch和path -func GetOwnerRepoBranchPath(URL string) (string, string, string, string, string, string, error) { +// parseOwnerRepoBranchPath parse owner、repo、branch and path from gitlab repo path +func parseOwnerRepoBranchPath(URL string) (string, string, string, string, string, string, error) { if !strings.Contains(URL, "https") && !strings.Contains(URL, "http") { - return "", "", "", "", "", "", fmt.Errorf("url is illegal") + return "", "", "", "", "", "", fmt.Errorf("url is illegal:%s", URL) } //适配公网的gitlab if strings.Contains(URL, "-") { @@ -130,3 +116,25 @@ func GetOwnerRepoBranchPath(URL string) (string, string, string, string, string, } return address, urlPathArray[3], urlPathArray[4], "", "", pathType, nil } + +// fillServiceRepoInfo ensures the value of repo is set +func fillServiceRepoInfo(svc *models.Service) error { + if svc.RepoOwner != "" && svc.RepoName != "" && svc.BranchName != "" && svc.LoadPath != "" { + svc.RepoNamespace = svc.GetRepoNamespace() + return nil + } + var err error + pathType := "" + _, svc.RepoOwner, svc.RepoNamespace, svc.BranchName, svc.LoadPath, pathType, err = parseOwnerRepoBranchPath(svc.SrcPath) + if err != nil { + return err + } + + if svc.Source == setting.SourceFromGitlab { + svc.LoadFromDir = pathType == "tree" + } else if svc.Source == setting.SourceFromGithub { + svc.LoadFromDir = true + } + svc.RepoNamespace = svc.GetRepoNamespace() + return nil +} diff --git a/pkg/microservice/aslan/core/common/service/webhook/client.go b/pkg/microservice/aslan/core/common/service/webhook/client.go index a51ee745500f82e21c8f2bfb784d43fbf5d721c3..1604bbcd8e9725b3005a98541a1fd6c075ef6e30 100644 --- a/pkg/microservice/aslan/core/common/service/webhook/client.go +++ b/pkg/microservice/aslan/core/common/service/webhook/client.go @@ -26,6 +26,8 @@ const ( PipelinePrefix = "pipeline-" ColliePrefix = "collie-" ServicePrefix = "service-" + TestingPrefix = "testing-" + ScannerPrefix = "scanning-" taskTimeoutSecond = 10 ) @@ -41,24 +43,28 @@ func NewClient() *client { } type task struct { - owner, repo, address, token, ref, ak, sk, region string - from string - add bool - err error - doneCh chan struct{} + ID int + owner, namespace, repo, address, token, ref, ak, sk, region string + from string + add, enableProxy bool + err error + doneCh chan struct{} } type TaskOption struct { - Name string - Owner string - Repo string - Address string - Token string - Ref string - From string - AK string - SK string - Region string + ID int + Name string + Owner string + Namespace string + Repo string + Address string + Token string + Ref string + From string + AK string + SK string + Region string + EnableProxy bool } func (c *client) AddWebHook(taskOption *TaskOption) error { @@ -67,17 +73,20 @@ func (c *client) AddWebHook(taskOption *TaskOption) error { } t := &task{ - owner: taskOption.Owner, - repo: taskOption.Repo, - address: taskOption.Address, - token: taskOption.Token, - ref: getFullReference(taskOption.Name, taskOption.Ref), - from: taskOption.From, - add: true, - ak: taskOption.AK, - sk: taskOption.SK, - region: taskOption.Region, - doneCh: make(chan struct{}), + ID: taskOption.ID, + owner: taskOption.Owner, + namespace: taskOption.Namespace, + repo: taskOption.Repo, + address: taskOption.Address, + token: taskOption.Token, + ref: getFullReference(taskOption.Name, taskOption.Ref), + from: taskOption.From, + add: true, + enableProxy: taskOption.EnableProxy, + ak: taskOption.AK, + sk: taskOption.SK, + region: taskOption.Region, + doneCh: make(chan struct{}), } select { @@ -101,17 +110,20 @@ func (c *client) RemoveWebHook(taskOption *TaskOption) error { } t := &task{ - owner: taskOption.Owner, - repo: taskOption.Repo, - address: taskOption.Address, - token: taskOption.Token, - ref: getFullReference(taskOption.Name, taskOption.Ref), - from: taskOption.From, - add: false, - ak: taskOption.AK, - sk: taskOption.SK, - region: taskOption.Region, - doneCh: make(chan struct{}), + ID: taskOption.ID, + owner: taskOption.Owner, + namespace: taskOption.Namespace, + repo: taskOption.Repo, + address: taskOption.Address, + token: taskOption.Token, + ref: getFullReference(taskOption.Name, taskOption.Ref), + from: taskOption.From, + add: false, + enableProxy: taskOption.EnableProxy, + ak: taskOption.AK, + sk: taskOption.SK, + region: taskOption.Region, + doneCh: make(chan struct{}), } select { diff --git a/pkg/microservice/aslan/core/common/service/webhook/controller.go b/pkg/microservice/aslan/core/common/service/webhook/controller.go index c9e809614072be5a615c4c659ffae62d9f19a3d1..62ec122e07ca9db636bb5ae53ab4e01e8f49e086 100644 --- a/pkg/microservice/aslan/core/common/service/webhook/controller.go +++ b/pkg/microservice/aslan/core/common/service/webhook/controller.go @@ -18,6 +18,7 @@ package webhook import ( "fmt" + "strings" "sync" "time" @@ -27,9 +28,10 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/codehub" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/gitee" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/github" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/gitlab" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/ilyshin" + codehostdb "github.com/koderover/zadig/pkg/microservice/systemconfig/core/codehost/repository/mongodb" "github.com/koderover/zadig/pkg/setting" "github.com/koderover/zadig/pkg/tool/log" ) @@ -51,7 +53,7 @@ var c *controller func webhookController() *controller { once.Do(func() { c = &controller{ - queue: make(chan *task, 100), + queue: make(chan *task, 500), logger: log.Logger(), } }) @@ -96,6 +98,7 @@ func (c *controller) processNextWorkItem() bool { logger := c.logger.With( zap.String("owner", t.owner), + zap.String("namespace", t.namespace), zap.String("repo", t.repo), zap.String("address", t.address), zap.String("ref", t.ref), @@ -115,6 +118,38 @@ func (c *controller) processNextWorkItem() bool { return true } +func ensureSafeDelete(ref, repoName, repoOwner, repoAddress string) (bool, error) { + // only handle webhooks created by service + if !strings.HasPrefix(ref, ServicePrefix) || !strings.HasSuffix(ref, "trigger") { + return true, nil + } + + serviceName := strings.TrimPrefix(ref, ServicePrefix) + serviceName = strings.TrimSuffix(serviceName, "-trigger") + + // find existing service with same name + services, err := mongodb.NewServiceColl().ListMaxRevisionsByProject(serviceName, setting.K8SDeployType) + if err != nil { + return false, err + } + + // check if the webhook reference points to other existing service + for _, service := range services { + if service.RepoName != repoName || service.RepoOwner != repoOwner { + continue + } + codeHostInfo, err := codehostdb.NewCodehostColl().GetCodeHostByID(service.CodehostID, false) + if err == nil { + if codeHostInfo.Address != repoAddress { + continue + } else { + return false, nil + } + } + } + return true, nil +} + func removeWebhook(t *task, logger *zap.Logger) { coll := mongodb.NewWebHookColl() var cl hookCreateDeleter @@ -122,25 +157,30 @@ func removeWebhook(t *task, logger *zap.Logger) { switch t.from { case setting.SourceFromGithub: - cl = github.NewClient(t.token, config.ProxyHTTPSAddr()) + cl = github.NewClient(t.token, config.ProxyHTTPSAddr(), t.enableProxy) case setting.SourceFromGitlab: - cl, err = gitlab.NewClient(t.address, t.token) + cl, err = gitlab.NewClient(t.ID, t.address, t.token, config.ProxyHTTPSAddr(), t.enableProxy) if err != nil { t.err = err t.doneCh <- struct{}{} return } case setting.SourceFromCodeHub: - cl = codehub.NewClient(t.ak, t.sk, t.region) - case setting.SourceFromIlyshin: - cl = ilyshin.NewClient(t.address, t.token) + cl = codehub.NewClient(t.ak, t.sk, t.region, config.ProxyHTTPSAddr(), t.enableProxy) + case setting.SourceFromGitee: + cl = gitee.NewClient(t.ID, t.token, config.ProxyHTTPSAddr(), t.enableProxy) default: t.err = fmt.Errorf("invaild source: %s", t.from) t.doneCh <- struct{}{} return } - webhook, err := coll.Find(t.owner, t.repo, t.address) + repoNamespace := t.namespace + if repoNamespace == "" { + repoNamespace = t.owner + } + + webhook, err := coll.Find(repoNamespace, t.repo, t.address) if err != nil { t.err = err t.doneCh <- struct{}{} @@ -149,27 +189,38 @@ func removeWebhook(t *task, logger *zap.Logger) { logger = logger.With(zap.String("hookID", webhook.HookID)) logger.Info("Removing webhook") - updated, err := coll.RemoveReference(t.owner, t.repo, t.address, t.ref) + + //ensure safe remove, same reference may be used by multiple services + safe, err := ensureSafeDelete(t.ref, t.repo, repoNamespace, t.address) if err != nil { t.err = err t.doneCh <- struct{}{} return } - if len(updated.References) == 0 { - logger.Info("Deleting webhook") - err = cl.DeleteWebHook(t.owner, t.repo, webhook.HookID) + if safe { + updated, err := coll.RemoveReference(repoNamespace, t.repo, t.address, t.ref) if err != nil { - logger.Error("Failed to delete webhook", zap.Error(err)) t.err = err t.doneCh <- struct{}{} return } - err = coll.Delete(t.owner, t.repo, t.address) - if err != nil { - logger.Error("Failed to delete webhook record in db", zap.Error(err)) - t.err = err + if len(updated.References) == 0 { + logger.Info("Deleting webhook") + err = cl.DeleteWebHook(repoNamespace, t.repo, webhook.HookID) + if err != nil { + logger.Error("Failed to delete webhook", zap.Error(err)) + t.err = err + t.doneCh <- struct{}{} + return + } + + err = coll.Delete(repoNamespace, t.repo, t.address) + if err != nil { + logger.Error("Failed to delete webhook record in db", zap.Error(err)) + t.err = err + } } } @@ -184,9 +235,9 @@ func addWebhook(t *task, logger *zap.Logger) { switch t.from { case setting.SourceFromGithub: - cl = github.NewClient(t.token, config.ProxyHTTPSAddr()) + cl = github.NewClient(t.token, config.ProxyHTTPSAddr(), t.enableProxy) case setting.SourceFromGitlab: - cl, err = gitlab.NewClient(t.address, t.token) + cl, err = gitlab.NewClient(t.ID, t.address, t.token, config.ProxyHTTPSAddr(), t.enableProxy) if err != nil { t.err = err t.doneCh <- struct{}{} @@ -194,17 +245,22 @@ func addWebhook(t *task, logger *zap.Logger) { } case setting.SourceFromCodeHub: - cl = codehub.NewClient(t.ak, t.sk, t.region) - case setting.SourceFromIlyshin: - cl = ilyshin.NewClient(t.address, t.token) + cl = codehub.NewClient(t.ak, t.sk, t.region, config.ProxyHTTPSAddr(), t.enableProxy) + case setting.SourceFromGitee: + cl = gitee.NewClient(t.ID, t.token, config.ProxyHTTPSAddr(), t.enableProxy) default: t.err = fmt.Errorf("invaild source: %s", t.from) t.doneCh <- struct{}{} return } + repoNamespace := t.namespace + if repoNamespace == "" { + repoNamespace = t.owner + } + logger.Info("Adding webhook") - created, err := coll.AddReferenceOrCreate(t.owner, t.repo, t.address, t.ref) + created, err := coll.AddReferenceOrCreate(repoNamespace, t.repo, t.address, t.ref) if err != nil || !created { t.err = err t.doneCh <- struct{}{} @@ -212,16 +268,16 @@ func addWebhook(t *task, logger *zap.Logger) { } logger.Info("Creating webhook") - hookID, err = cl.CreateWebHook(t.owner, t.repo) + hookID, err = cl.CreateWebHook(repoNamespace, t.repo) if err != nil { t.err = err logger.Error("Failed to create webhook", zap.Error(err)) - if err = coll.Delete(t.owner, t.repo, t.address); err != nil { + if err = coll.Delete(repoNamespace, t.repo, t.address); err != nil { logger.Error("Failed to delete webhook record in db", zap.Error(err)) } } else { if hookID != "" { - if err = coll.Update(t.owner, t.repo, t.address, hookID); err != nil { + if err = coll.Update(repoNamespace, t.repo, t.address, hookID); err != nil { t.err = err logger.Error("Failed to update webhook", zap.Error(err)) } diff --git a/pkg/microservice/aslan/core/common/service/webhook/types.go b/pkg/microservice/aslan/core/common/service/webhook/types.go index 4bf88744ee06bfb132f650ddf1d1a06a2656e33e..c89dde932322a442ad635ed68775aaaad9433f32 100644 --- a/pkg/microservice/aslan/core/common/service/webhook/types.go +++ b/pkg/microservice/aslan/core/common/service/webhook/types.go @@ -17,6 +17,6 @@ limitations under the License. package webhook type WebHook struct { - Owner, Repo, Address, Name string - CodeHostID int + Owner, Namespace, Repo, Address, Name string + CodeHostID int } diff --git a/pkg/microservice/aslan/core/common/service/wechat/service.go b/pkg/microservice/aslan/core/common/service/wechat/service.go deleted file mode 100644 index 579df39f611fe716cfac5b0e5284ee0b5de1e522..0000000000000000000000000000000000000000 --- a/pkg/microservice/aslan/core/common/service/wechat/service.go +++ /dev/null @@ -1,353 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package wechat - -import ( - "bytes" - "fmt" - "strings" - "text/template" - "time" - - "k8s.io/apimachinery/pkg/util/sets" - - configbase "github.com/koderover/zadig/pkg/config" - "github.com/koderover/zadig/pkg/microservice/aslan/config" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" - "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/tool/httpclient" - "github.com/koderover/zadig/pkg/tool/log" -) - -const ( - msgType = "markdown" - markdownColorInfo = "info" - markdownColorComment = "comment" - markdownColorWarning = "warning" - singleInfo = "single" - multiInfo = "multi" - dingDingType = "dingding" - feiShuType = "feishu" -) - -//wechat -type Messsage struct { - MsgType string `json:"msgtype"` - Text *Text `json:"markdown"` -} - -type Text struct { - Content string `json:"content"` -} - -//dingding -type DingDingMessage struct { - MsgType string `json:"msgtype"` - MarkDown *DingDingMarkDown `json:"markdown"` - At *DingDingAt `json:"at"` -} - -type DingDingMarkDown struct { - Title string `json:"title"` - Text string `json:"text"` -} - -type DingDingAt struct { - AtMobiles []string `json:"atMobiles"` - IsAtAll bool `json:"isAtAll"` -} - -type FeiShuMessage struct { - Title string `json:"title"` - Text string `json:"text"` -} - -type FeiShuMessageV2 struct { - MsgType string `json:"msg_type"` - Content FeiShuContentV2 `json:"content"` -} - -type FeiShuContentV2 struct { - Text string `json:"text"` -} - -type Service struct { - proxyColl *mongodb.ProxyColl - workflowColl *mongodb.WorkflowColl - pipelineColl *mongodb.PipelineColl -} - -func NewWeChatClient() *Service { - return &Service{ - proxyColl: mongodb.NewProxyColl(), - workflowColl: mongodb.NewWorkflowColl(), - pipelineColl: mongodb.NewPipelineColl(), - } -} - -type wechatNotification struct { - Task *task.Task `json:"task"` - BaseURI string `json:"base_uri"` - IsSingle bool `json:"is_single"` - WebHookType string `json:"web_hook_type"` - TotalTime int64 `json:"total_time"` - AtMobiles []string `json:"atMobiles"` - IsAtAll bool `json:"is_at_all"` -} - -func (w *Service) SendMessageRequest(uri string, message interface{}) ([]byte, error) { - c := httpclient.New() - - // 使用代理 - proxies, _ := w.proxyColl.List(&mongodb.ProxyArgs{}) - if len(proxies) != 0 && proxies[0].EnableApplicationProxy { - c.SetProxy(proxies[0].GetProxyURL()) - fmt.Printf("send message is using proxy:%s\n", proxies[0].GetProxyURL()) - } - - res, err := c.Post(uri, httpclient.SetBody(message)) - if err != nil { - return nil, err - } - - return res.Body(), nil -} - -func (w *Service) SendWechatMessage(task *task.Task) error { - var ( - uri = "" - content = "" - webHookType = "" - atMobiles []string - isAtAll bool - ) - if task.Type == config.SingleType { - resp, err := w.pipelineColl.Find(&mongodb.PipelineFindOption{Name: task.PipelineName}) - if err != nil { - log.Errorf("Pipeline find err :%v", err) - return err - } - if resp.NotifyCtl == nil { - log.Infof("pipeline notifyCtl is not set!") - return nil - } - if resp.NotifyCtl.Enabled && sets.NewString(resp.NotifyCtl.NotifyTypes...).Has(string(task.Status)) { - webHookType = resp.NotifyCtl.WebHookType - if webHookType == dingDingType { - uri = resp.NotifyCtl.DingDingWebHook - atMobiles = resp.NotifyCtl.AtMobiles - isAtAll = resp.NotifyCtl.IsAtAll - } else if webHookType == feiShuType { - uri = resp.NotifyCtl.FeiShuWebHook - } else { - uri = resp.NotifyCtl.WeChatWebHook - } - content, err = w.createNotifyBody(&wechatNotification{ - Task: task, - BaseURI: configbase.SystemAddress(), - IsSingle: true, - WebHookType: webHookType, - TotalTime: time.Now().Unix() - task.StartTime, - AtMobiles: atMobiles, - IsAtAll: isAtAll, - }) - if err != nil { - log.Errorf("pipeline CreateNotifyBody err :%v", err) - return err - } - } - } else if task.Type == config.WorkflowType { - resp, err := w.workflowColl.Find(task.PipelineName) - if err != nil { - log.Errorf("Workflow find err :%v", err) - return err - } - if resp.NotifyCtl == nil { - log.Infof("Workflow notifyCtl is not set!") - return nil - } - if resp.NotifyCtl.Enabled && sets.NewString(resp.NotifyCtl.NotifyTypes...).Has(string(task.Status)) { - webHookType = resp.NotifyCtl.WebHookType - if webHookType == dingDingType { - uri = resp.NotifyCtl.DingDingWebHook - atMobiles = resp.NotifyCtl.AtMobiles - isAtAll = resp.NotifyCtl.IsAtAll - } else if webHookType == feiShuType { - uri = resp.NotifyCtl.FeiShuWebHook - } else { - uri = resp.NotifyCtl.WeChatWebHook - } - content, err = w.createNotifyBody(&wechatNotification{ - Task: task, - BaseURI: configbase.SystemAddress(), - IsSingle: false, - WebHookType: webHookType, - TotalTime: time.Now().Unix() - task.StartTime, - AtMobiles: atMobiles, - IsAtAll: isAtAll, - }) - if err != nil { - log.Errorf("workflow CreateNotifyBody err :%v", err) - return err - } - } - } - - if uri != "" && content != "" { - if webHookType == dingDingType { - message := &DingDingMessage{ - MsgType: msgType, - MarkDown: &DingDingMarkDown{ - Title: "工作流状态", - Text: content, - }, - } - if len(atMobiles) > 0 { - message.At = &DingDingAt{ - AtMobiles: atMobiles, - IsAtAll: false, - } - } else { - message.At = &DingDingAt{ - IsAtAll: true, - } - } - - _, err := w.SendMessageRequest(uri, message) - if err != nil { - log.Errorf("SendDingDingMessageRequest err : %v", err) - return err - } - - } else if webHookType == feiShuType { - var message interface{} - message = &FeiShuMessage{ - Title: "工作流状态", - Text: content, - } - if strings.Contains(uri, "bot/v2/hook") { - message = &FeiShuMessageV2{ - MsgType: "text", - Content: FeiShuContentV2{ - Text: content, - }, - } - } - _, err := w.SendMessageRequest(uri, message) - if err != nil { - log.Errorf("SendFeiShuMessageRequest err : %v", err) - return err - } - } else { - message := &Messsage{ - MsgType: msgType, - Text: &Text{ - Content: content, - }, - } - _, err := w.SendMessageRequest(uri, message) - if err != nil { - log.Errorf("SendWeChatMessageRequest err : %v", err) - return err - } - } - - } - return nil -} - -func (w *Service) createNotifyBody(weChatNotification *wechatNotification) (content string, err error) { - tmplSource := "{{if eq .WebHookType \"feishu\"}}触发的工作流: {{.BaseURI}}/v1/projects/detail/{{.Task.ProductName}}/pipelines/{{ isSingle .IsSingle }}/{{.Task.PipelineName}}/{{.Task.TaskID}}{{else}}#### 触发的工作流: [{{.Task.PipelineName}}#{{.Task.TaskID}}]({{.BaseURI}}/v1/projects/detail/{{.Task.ProductName}}/pipelines/{{ isSingle .IsSingle }}/{{.Task.PipelineName}}/{{.Task.TaskID}}){{end}} \n" + - "- 状态: {{if eq .WebHookType \"feishu\"}}{{.Task.Status}}{{else}}{{.Task.Status}}{{end}} \n" + - "- 创建人：{{.Task.TaskCreator}} \n" + - "- 总运行时长：{{ .TotalTime}} 秒 \n" - - testNames := getHTMLTestReport(weChatNotification.Task) - if len(testNames) != 0 { - tmplSource += "- 测试报告：\n" - } - - for _, testName := range testNames { - url := fmt.Sprintf("{{.BaseURI}}/api/aslan/testing/report?pipelineName={{.Task.PipelineName}}&pipelineType={{.Task.Type}}&taskID={{.Task.TaskID}}&testName=%s\n", testName) - if weChatNotification.WebHookType == feiShuType { - tmplSource += url - continue - } - tmplSource += fmt.Sprintf("[%s](%s)\n", url, url) - } - - if weChatNotification.WebHookType == dingDingType { - if len(weChatNotification.AtMobiles) > 0 && !weChatNotification.IsAtAll { - tmplSource = fmt.Sprintf("%s - 相关人员：@%s \n", tmplSource, strings.Join(weChatNotification.AtMobiles, "@")) - } - } - - tmpl := template.Must(template.New("notify").Funcs(template.FuncMap{ - "getColor": func(status config.Status) string { - if status == config.StatusPassed { - return markdownColorInfo - } else if status == config.StatusTimeout || status == config.StatusCancelled { - return markdownColorComment - } else if status == config.StatusFailed { - return markdownColorWarning - } - return markdownColorComment - }, - "isSingle": func(isSingle bool) string { - if isSingle { - return singleInfo - } - return multiInfo - }, - }).Parse(tmplSource)) - buffer := bytes.NewBufferString("") - - if err = tmpl.Execute(buffer, &weChatNotification); err != nil { - return - } - - return buffer.String(), nil -} - -func getHTMLTestReport(task *task.Task) []string { - if task.Type != config.WorkflowType { - return nil - } - - testNames := make([]string, 0) - for _, stage := range task.Stages { - if stage.TaskType != config.TaskTestingV2 { - continue - } - - for testName, subTask := range stage.SubTasks { - testInfo, err := base.ToTestingTask(subTask) - if err != nil { - log.Errorf("parse testInfo failed, err:%s", err) - continue - } - - if testInfo.JobCtx.TestType == setting.FunctionTest && testInfo.JobCtx.TestReportPath != "" { - testNames = append(testNames, testName) - } - } - } - - return testNames -} diff --git a/pkg/microservice/aslan/core/common/service/workflow.go b/pkg/microservice/aslan/core/common/service/workflow.go index 325a3f3dec3a9dc7f5c3a54010cd05c9cb1e5857..20b4971de64a9bbbc098d78b5fd8051d1b6119ca 100644 --- a/pkg/microservice/aslan/core/common/service/workflow.go +++ b/pkg/microservice/aslan/core/common/service/workflow.go @@ -17,6 +17,7 @@ limitations under the License. package service import ( + "encoding/json" "fmt" "sync" @@ -27,15 +28,18 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/collaboration" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/gerrit" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/nsq" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/webhook" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/codehost" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/types" ) func DeleteWorkflows(productName, requestID string, log *zap.SugaredLogger) error { - workflows, err := mongodb.NewWorkflowColl().List(&mongodb.ListWorkflowOption{ProductName: productName}) + workflows, err := mongodb.NewWorkflowColl().List(&mongodb.ListWorkflowOption{Projects: []string{productName}}) if err != nil { log.Errorf("Workflow.List error: %v", err) return fmt.Errorf("DeleteWorkflows productName %s Workflow.List error: %v", productName, err) @@ -54,6 +58,19 @@ func DeleteWorkflows(productName, requestID string, log *zap.SugaredLogger) erro } func DeleteWorkflow(workflowName, requestID string, isDeletingProductTmpl bool, log *zap.SugaredLogger) error { + // query workflow before deleting, used to delete gerrit webhook + workflow, err := mongodb.NewWorkflowColl().Find(workflowName) + if err != nil { + log.Errorf("Workflow.Find error: %v", err) + return err + } + workflowCMMap, err := collaboration.GetWorkflowCMMap([]string{workflow.ProductTmplName}, log) + if err != nil { + return err + } + if cmSets, ok := workflowCMMap[collaboration.BuildWorkflowCMMapKey(workflow.ProductTmplName, workflowName)]; ok { + return fmt.Errorf("this is a base workflow, collaborations:%v is related", cmSets.List()) + } taskQueue, err := mongodb.NewQueueColl().List(&mongodb.ListQueueOption{}) if err != nil { log.Errorf("List queued task error: %v", err) @@ -68,13 +85,6 @@ func DeleteWorkflow(workflowName, requestID string, isDeletingProductTmpl bool, } } - // 在删除前，先将workflow查出来，用于删除gerrit webhook - workflow, err := mongodb.NewWorkflowColl().Find(workflowName) - if err != nil { - log.Errorf("Workflow.Find error: %v", err) - return e.ErrDeleteWorkflow.AddDesc(err.Error()) - } - if !isDeletingProductTmpl { prod, err := template.NewProductColl().Find(workflow.ProductTmplName) if err != nil { @@ -91,6 +101,11 @@ func DeleteWorkflow(workflowName, requestID string, isDeletingProductTmpl bool, log.Errorf("Failed to process webhook, err: %s", err) } + err = DisableCronjobForWorkflow(workflow) + if err != nil { + log.Errorf("Failed to stop cronjob for workflow: %s, error: %s", workflow.Name, err) + } + go gerrit.DeleteGerritWebhook(workflow, log) //删除所属的所有定时任务 @@ -112,7 +127,7 @@ func DeleteWorkflow(workflowName, requestID string, isDeletingProductTmpl bool, log.Errorf("PipelineTaskV2.DeleteByPipelineName error: %v", err) } - if deliveryVersions, err := mongodb.NewDeliveryVersionColl().Find(&mongodb.DeliveryVersionArgs{OrgID: 1, WorkflowName: workflowName}); err == nil { + if deliveryVersions, err := mongodb.NewDeliveryVersionColl().Find(&mongodb.DeliveryVersionArgs{WorkflowName: workflowName}); err == nil { for _, deliveryVersion := range deliveryVersions { if err := mongodb.NewDeliveryVersionColl().Delete(deliveryVersion.ID.Hex()); err != nil { log.Errorf("DeleteWorkflow.DeliveryVersion.Delete error: %v", err) @@ -147,6 +162,32 @@ func DeleteWorkflow(workflowName, requestID string, isDeletingProductTmpl bool, return nil } +func DisableCronjobForWorkflow(workflow *models.Workflow) error { + disableIDList := make([]string, 0) + payload := &CronjobPayload{ + Name: workflow.Name, + JobType: config.WorkflowCronjob, + Action: setting.TypeEnableCronjob, + } + if workflow.ScheduleEnabled { + jobList, err := mongodb.NewCronjobColl().List(&mongodb.ListCronjobParam{ + ParentName: workflow.Name, + ParentType: config.WorkflowCronjob, + }) + if err != nil { + return err + } + + for _, job := range jobList { + disableIDList = append(disableIDList, job.ID.Hex()) + } + payload.DeleteList = disableIDList + } + + pl, _ := json.Marshal(payload) + return nsq.Publish(setting.TopicCronjob, pl) +} + func ProcessWebhook(updatedHooks, currentHooks interface{}, name string, logger *zap.SugaredLogger) error { currentSet := toHookSet(currentHooks) updatedSet := toHookSet(updatedHooks) @@ -167,7 +208,7 @@ func ProcessWebhook(updatedHooks, currentHooks interface{}, name string, logger wg.Add(1) go func(wh hookItem) { defer wg.Done() - ch, err := codehost.GetCodeHostInfoByID(wh.codeHostID) + ch, err := systemconfig.New().GetRawCodeHost(wh.codeHostID) if err != nil { logger.Errorf("Failed to get codeHost by id %d, err: %s", wh.codeHostID, err) errs = multierror.Append(errs, err) @@ -175,18 +216,21 @@ func ProcessWebhook(updatedHooks, currentHooks interface{}, name string, logger } switch ch.Type { - case setting.SourceFromGithub, setting.SourceFromGitlab, setting.SourceFromCodeHub, setting.SourceFromIlyshin: + case setting.SourceFromGithub, setting.SourceFromGitlab, setting.SourceFromCodeHub, setting.SourceFromGitee: err = webhook.NewClient().RemoveWebHook(&webhook.TaskOption{ - Name: wh.name, - Owner: wh.owner, - Repo: wh.repo, - Address: ch.Address, - Token: ch.AccessToken, - AK: ch.AccessKey, - SK: ch.SecretKey, - Region: ch.Region, - Ref: name, - From: ch.Type, + ID: ch.ID, + Name: wh.name, + Owner: wh.owner, + Namespace: wh.namespace, + Repo: wh.repo, + Address: ch.Address, + Token: ch.AccessToken, + AK: ch.AccessKey, + SK: ch.SecretKey, + Region: ch.Region, + EnableProxy: ch.EnableProxy, + Ref: name, + From: ch.Type, }) if err != nil { logger.Errorf("Failed to remove %s webhook %+v, err: %s", ch.Type, wh, err) @@ -201,7 +245,7 @@ func ProcessWebhook(updatedHooks, currentHooks interface{}, name string, logger wg.Add(1) go func(wh hookItem) { defer wg.Done() - ch, err := codehost.GetCodeHostInfoByID(wh.codeHostID) + ch, err := systemconfig.New().GetCodeHost(wh.codeHostID) if err != nil { logger.Errorf("Failed to get codeHost by id %d, err: %s", wh.codeHostID, err) errs = multierror.Append(errs, err) @@ -209,18 +253,20 @@ func ProcessWebhook(updatedHooks, currentHooks interface{}, name string, logger } switch ch.Type { - case setting.SourceFromGithub, setting.SourceFromGitlab, setting.SourceFromCodeHub, setting.SourceFromIlyshin: + case setting.SourceFromGithub, setting.SourceFromGitlab, setting.SourceFromCodeHub, setting.SourceFromGitee: err = webhook.NewClient().AddWebHook(&webhook.TaskOption{ - Name: wh.name, - Owner: wh.owner, - Repo: wh.repo, - Address: ch.Address, - Token: ch.AccessToken, - Ref: name, - AK: ch.AccessKey, - SK: ch.SecretKey, - Region: ch.Region, - From: ch.Type, + ID: ch.ID, + Name: wh.name, + Owner: wh.owner, + Namespace: wh.namespace, + Repo: wh.repo, + Address: ch.Address, + Token: ch.AccessToken, + Ref: name, + AK: ch.AccessKey, + SK: ch.SecretKey, + Region: ch.Region, + From: ch.Type, }) if err != nil { logger.Errorf("Failed to add %s webhook %+v, err: %s", ch.Type, wh, err) @@ -243,10 +289,11 @@ func toHookSet(hooks interface{}) HookSet { for _, h := range hs { res.Insert(hookItem{ hookUniqueID: hookUniqueID{ - name: h.MainRepo.Name, - owner: h.MainRepo.RepoOwner, - repo: h.MainRepo.RepoName, - source: h.MainRepo.Source, + name: h.MainRepo.Name, + owner: h.MainRepo.RepoOwner, + namespace: h.MainRepo.GetRepoNamespace(), + repo: h.MainRepo.RepoName, + source: h.MainRepo.Source, }, codeHostID: h.MainRepo.CodehostID, }) @@ -255,9 +302,10 @@ func toHookSet(hooks interface{}) HookSet { for _, h := range hs { res.Insert(hookItem{ hookUniqueID: hookUniqueID{ - name: h.Name, - owner: h.Owner, - repo: h.Repo, + name: h.Name, + owner: h.Owner, + namespace: h.Owner, // webhooks for pipelines, no need to handler anymore + repo: h.Repo, }, codeHostID: h.CodehostID, }) @@ -266,13 +314,37 @@ func toHookSet(hooks interface{}) HookSet { for _, h := range hs { res.Insert(hookItem{ hookUniqueID: hookUniqueID{ - name: h.Name, - owner: h.Owner, - repo: h.Repo, + name: h.Name, + owner: h.Owner, + namespace: h.Namespace, + repo: h.Repo, }, codeHostID: h.CodeHostID, }) } + case []*models.TestingHook: + for _, h := range hs { + res.Insert(hookItem{ + hookUniqueID: hookUniqueID{ + name: h.MainRepo.Name, + owner: h.MainRepo.RepoOwner, + namespace: h.MainRepo.GetRepoNamespace(), + repo: h.MainRepo.RepoName, + }, + codeHostID: h.MainRepo.CodehostID, + }) + } + case []*types.ScanningHook: + for _, h := range hs { + res.Insert(hookItem{ + hookUniqueID: hookUniqueID{ + name: h.RepoName, + owner: h.RepoOwner, + repo: h.RepoName, + }, + codeHostID: h.CodehostID, + }) + } } return res diff --git a/pkg/microservice/aslan/core/common/service/workflow_v3.go b/pkg/microservice/aslan/core/common/service/workflow_v3.go new file mode 100644 index 0000000000000000000000000000000000000000..16ae171b0c3ad151861ab860234cf5fad6bc7773 --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/workflow_v3.go @@ -0,0 +1,77 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "fmt" + + "go.uber.org/zap" + + "github.com/hashicorp/go-multierror" + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +func DeleteWorkflowV3s(productName, requestID string, log *zap.SugaredLogger) error { + workflowV3s, _, err := mongodb.NewWorkflowV3Coll().List(&mongodb.ListWorkflowV3Option{ProjectName: productName}, 0, 0) + if err != nil { + log.Errorf("WorkflowV3.List error: %s", err) + return fmt.Errorf("DeleteWorkflowV3s productName %s WorkflowV3.List error: %s", productName, err) + } + errList := new(multierror.Error) + for _, workflowV3 := range workflowV3s { + if err = DeleteWorkflowV3(workflowV3.ID.Hex(), workflowV3.Name, requestID, log); err != nil { + errList = multierror.Append(errList, fmt.Errorf("productName %s workflowV3 delete %s error: %s", productName, workflowV3.Name, err)) + } + } + if err := errList.ErrorOrNil(); err != nil { + log.Error(err) + return err + } + return nil +} + +func DeleteWorkflowV3(workflowID, workflowName, requestID string, log *zap.SugaredLogger) error { + taskQueue, err := mongodb.NewQueueColl().List(&mongodb.ListQueueOption{}) + if err != nil { + log.Errorf("List queued task error: %s", err) + return e.ErrDeletePipeline.AddErr(err) + } + // 当task还在运行时，先取消任务 + for _, task := range taskQueue { + if task.PipelineName == workflowName && task.Type == config.WorkflowTypeV3 { + if err = CancelWorkflowTaskV3("system", task.PipelineName, task.TaskID, config.WorkflowTypeV3, requestID, log); err != nil { + log.Errorf("task still running, cancel pipelineV3 %s task %d", task.PipelineName, task.TaskID) + } + } + } + + if err := mongodb.NewWorkflowV3Coll().DeleteByID(workflowID); err != nil { + log.Errorf("WorkflowV3.delete error: %s", err) + return e.ErrDeleteWorkflow.AddDesc(err.Error()) + } + + if err := mongodb.NewTaskColl().DeleteByPipelineNameAndType(workflowName, config.WorkflowTypeV3); err != nil { + log.Errorf("PipelineTaskV2.DeleteByPipelineName error: %s", err) + } + + if err := mongodb.NewCounterColl().Delete("WorkflowTaskV3:" + workflowName); err != nil { + log.Errorf("Counter.Delete error: %s", err) + } + return nil +} diff --git a/pkg/microservice/aslan/core/common/service/workflow_v4.go b/pkg/microservice/aslan/core/common/service/workflow_v4.go new file mode 100644 index 0000000000000000000000000000000000000000..a716cbc5fc10971bb685e39c4317b4b8d582946a --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/workflow_v4.go @@ -0,0 +1,125 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "fmt" + + "github.com/hashicorp/go-multierror" + "go.uber.org/zap" + + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/workflowcontroller" + "github.com/koderover/zadig/pkg/tool/crypto" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/log" +) + +func DeleteWorkflowV4sByProjectName(projectName string, log *zap.SugaredLogger) error { + workflowV4s, _, err := mongodb.NewWorkflowV4Coll().List(&mongodb.ListWorkflowV4Option{ProjectName: projectName}, 0, 0) + if err != nil { + log.Errorf("WorkflowV4.List error: %s", err) + return fmt.Errorf("DeleteWorkflowV4s productName %s WorkflowV4.List error: %s", projectName, err) + } + errList := new(multierror.Error) + for _, workflowV4 := range workflowV4s { + if err = DeleteWorkflowV4(workflowV4.Name, log); err != nil { + errList = multierror.Append(errList, fmt.Errorf("productName %s workflowV4 delete %s error: %s", projectName, workflowV4.Name, err)) + } + } + if err := errList.ErrorOrNil(); err != nil { + log.Error(err) + return err + } + return nil +} + +func DeleteWorkflowV4(name string, logger *zap.SugaredLogger) error { + taskQueue, err := mongodb.NewWorkflowQueueColl().List(&mongodb.ListWorfklowQueueOption{}) + if err != nil { + log.Errorf("List queued task error: %s", err) + return e.ErrDeletePipeline.AddErr(err) + } + // when task still running,cancel task first. + for _, task := range taskQueue { + if task.WorkflowName == name { + if err = workflowcontroller.CancelWorkflowTask("system", task.WorkflowName, task.TaskID, logger); err != nil { + log.Errorf("task still running, cancel pipelineV4 %s task %d", task.WorkflowName, task.TaskID) + } + } + } + workflow, err := mongodb.NewWorkflowV4Coll().Find(name) + if err != nil { + logger.Errorf("Failed to delete WorkflowV4: %s, the error is: %v", name, err) + return e.ErrDeleteWorkflow.AddErr(err) + } + if err := mongodb.NewWorkflowV4Coll().DeleteByID(workflow.ID.Hex()); err != nil { + logger.Errorf("Failed to delete WorkflowV4: %s, the error is: %v", name, err) + return e.ErrDeleteWorkflow.AddErr(err) + } + if err := mongodb.NewworkflowTaskv4Coll().DeleteByWorkflowName(name); err != nil { + logger.Errorf("Failed to delete WorkflowV4 task: %s, the error is: %v", name, err) + return e.ErrDeleteWorkflow.AddErr(err) + } + if err := mongodb.NewCounterColl().Delete("WorkflowTaskV4:" + name); err != nil { + log.Errorf("Counter.Delete error: %s", err) + } + return nil +} + +func EncryptKeyVals(encryptedKey string, kvs []*commonmodels.KeyVal, logger *zap.SugaredLogger) error { + if encryptedKey == "" { + return nil + } + aesKey, err := GetAesKeyFromEncryptedKey(encryptedKey, logger) + if err != nil { + log.Errorf("EncypteKeyVals GetAesKeyFromEncryptedKey err:%v", err) + return err + } + for _, kv := range kvs { + if kv.IsCredential { + kv.Value, err = crypto.AesEncryptByKey(kv.Value, aesKey.PlainText) + if err != nil { + log.Errorf("aes encrypt by key err:%v", err) + return err + } + } + } + return nil +} + +func EncryptParams(encryptedKey string, params []*commonmodels.Params, logger *zap.SugaredLogger) error { + if encryptedKey == "" { + return nil + } + aesKey, err := GetAesKeyFromEncryptedKey(encryptedKey, logger) + if err != nil { + log.Errorf("EncyptParams GetAesKeyFromEncryptedKey err:%v", err) + return err + } + for _, param := range params { + if param.IsCredential { + param.Value, err = crypto.AesEncryptByKey(param.Value, aesKey.PlainText) + if err != nil { + log.Errorf("aes encrypt by key err:%v", err) + return err + } + } + } + return nil +} diff --git a/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job.go b/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job.go new file mode 100644 index 0000000000000000000000000000000000000000..9499b6939760ed2c17e105719fd3b25c8e71f14d --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job.go @@ -0,0 +1,188 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package jobcontroller + +import ( + "context" + "fmt" + "io" + "os" + "strings" + "sync" + "time" + + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/util/rand" +) + +type JobCtl interface { + Run(ctx context.Context) + Wait(ctx context.Context) + Complete(ctx context.Context) +} + +func runJob(ctx context.Context, job *commonmodels.JobTask, workflowCtx *commonmodels.WorkflowTaskCtx, logger *zap.SugaredLogger, ack func()) { + job.Status = config.StatusRunning + job.StartTime = time.Now().Unix() + ack() + // set default timeout + if job.Properties.Timeout <= 0 { + job.Properties.Timeout = 600 + } + // set default resource + if job.Properties.ResourceRequest == setting.Request("") { + job.Properties.ResourceRequest = setting.MinRequest + } + // set default resource + if job.Properties.ClusterID == "" { + job.Properties.ClusterID = setting.LocalClusterID + } + // init step configration. + if err := stepcontroller.PrepareSteps(ctx, workflowCtx, &job.Properties.Paths, job.Steps, logger); err != nil { + logger.Error(err) + job.Error = err.Error() + job.Status = config.StatusFailed + job.EndTime = time.Now().Unix() + logger.Infof("finish job: %s,status: %s", job.Name, job.Status) + ack() + return + } + + logger.Infof("start job: %s,status: %s", job.Name, job.Status) + defer func() { + if err := stepcontroller.SummarizeSteps(ctx, workflowCtx, &job.Properties.Paths, job.Steps, logger); err != nil { + logger.Error(err) + job.Error = err.Error() + job.Status = config.StatusFailed + } + job.EndTime = time.Now().Unix() + logger.Infof("finish job: %s,status: %s", job.Name, job.Status) + ack() + }() + var jobCtl JobCtl + switch job.JobType { + case string(config.JobZadigDeploy): + fallthrough + case string(config.JobDeploy): + // do deploy inside aslan instead of jobexecutor. + status, err := stepcontroller.RunSteps(ctx, workflowCtx, &job.Properties.Paths, job.Steps, logger) + job.Status = status + if err != nil { + logger.Error(err) + job.Error = err.Error() + } + return + case string(config.JobPlugin): + jobCtl = NewPluginsJobCtl(job, workflowCtx, ack, logger) + default: + jobCtl = NewFreestyleJobCtl(job, workflowCtx, ack, logger) + } + + jobCtl.Run(ctx) + jobCtl.Wait(ctx) + jobCtl.Complete(ctx) +} + +func RunJobs(ctx context.Context, jobs []*commonmodels.JobTask, workflowCtx *commonmodels.WorkflowTaskCtx, concurrency int, logger *zap.SugaredLogger, ack func()) { + jobPool := NewPool(ctx, jobs, workflowCtx, concurrency, logger, ack) + jobPool.Run() +} + +// Pool is a worker group that runs a number of tasks at a +// configured concurrency. +type Pool struct { + Jobs []*commonmodels.JobTask + workflowCtx *commonmodels.WorkflowTaskCtx + concurrency int + jobsChan chan *commonmodels.JobTask + logger *zap.SugaredLogger + ack func() + ctx context.Context + wg sync.WaitGroup +} + +// NewPool initializes a new pool with the given tasks and +// at the given concurrency. +func NewPool(ctx context.Context, jobs []*commonmodels.JobTask, workflowCtx *commonmodels.WorkflowTaskCtx, concurrency int, logger *zap.SugaredLogger, ack func()) *Pool { + return &Pool{ + Jobs: jobs, + concurrency: concurrency, + workflowCtx: workflowCtx, + jobsChan: make(chan *commonmodels.JobTask), + logger: logger, + ack: ack, + ctx: ctx, + } +} + +// Run runs all job within the pool and blocks until it's +// finished. +func (p *Pool) Run() { + for i := 0; i < p.concurrency; i++ { + go p.work() + } + + p.wg.Add(len(p.Jobs)) + for _, task := range p.Jobs { + p.jobsChan <- task + } + + // all workers return + close(p.jobsChan) + + p.wg.Wait() +} + +// The work loop for any single goroutine. +func (p *Pool) work() { + for job := range p.jobsChan { + runJob(p.ctx, job, p.workflowCtx, p.logger, p.ack) + p.wg.Done() + } +} + +func saveFile(src io.Reader, localFile string) error { + out, err := os.Create(localFile) + if err != nil { + return err + } + + defer out.Close() + + _, err = io.Copy(out, src) + return err +} + +func getJobName(workflowName string, taskID int64) string { + // max lenth of workflowName was 32, so job name was unique in one task. + base := strings.Replace( + strings.ToLower( + fmt.Sprintf( + "%s-%d-", + workflowName, + taskID, + ), + ), + "_", "-", -1, + ) + return rand.GenerateName(base) +} diff --git a/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job_freestyle.go b/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job_freestyle.go new file mode 100644 index 0000000000000000000000000000000000000000..d200afd794208632cb28d341594a83a1ee520092 --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job_freestyle.go @@ -0,0 +1,268 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package jobcontroller + +import ( + "context" + "fmt" + "strings" + "time" + + "go.uber.org/zap" + "gopkg.in/yaml.v3" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + crClient "sigs.k8s.io/controller-runtime/pkg/client" + + zadigconfig "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/dockerhost" + krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client" + "github.com/koderover/zadig/pkg/tool/kube/updater" +) + +const ( + DindServer = "dind" + KoderoverAgentNamespace = "koderover-agent" +) + +type FreestyleJobCtl struct { + job *commonmodels.JobTask + jobName string + workflowCtx *commonmodels.WorkflowTaskCtx + logger *zap.SugaredLogger + kubeclient crClient.Client + clientset kubernetes.Interface + restConfig *rest.Config + paths *string + ack func() +} + +func NewFreestyleJobCtl(job *commonmodels.JobTask, workflowCtx *commonmodels.WorkflowTaskCtx, ack func(), logger *zap.SugaredLogger) *FreestyleJobCtl { + paths := "" + return &FreestyleJobCtl{ + job: job, + workflowCtx: workflowCtx, + logger: logger, + ack: ack, + paths: &paths, + jobName: getJobName(workflowCtx.WorkflowName, workflowCtx.TaskID), + } +} + +func (c *FreestyleJobCtl) Run(ctx context.Context) { + // get kube client + hubServerAddr := config.HubServerAddress() + switch c.job.Properties.ClusterID { + case setting.LocalClusterID: + c.job.Properties.Namespace = zadigconfig.Namespace() + c.kubeclient = krkubeclient.Client() + c.clientset = krkubeclient.Clientset() + c.restConfig = krkubeclient.RESTConfig() + default: + c.job.Properties.Namespace = setting.AttachedClusterNamespace + + crClient, clientset, restConfig, err := GetK8sClients(hubServerAddr, c.job.Properties.ClusterID) + if err != nil { + c.job.Status = config.StatusFailed + c.job.Error = err.Error() + c.job.EndTime = time.Now().Unix() + return + } + c.kubeclient = crClient + c.clientset = clientset + c.restConfig = restConfig + } + + // decide which docker host to use. + // TODO: do not use code in warpdrive moudule, should move to a public place + dockerhosts := dockerhost.NewDockerHosts(hubServerAddr, c.logger) + c.job.Properties.DockerHost = dockerhosts.GetBestHost(dockerhost.ClusterID(c.job.Properties.ClusterID), "") + + // not local cluster + var ( + replaceDindServer = "." + DindServer + dockerHost = "" + ) + + if c.job.Properties.ClusterID != "" && c.job.Properties.ClusterID != setting.LocalClusterID { + if strings.Contains(c.job.Properties.DockerHost, config.Namespace()) { + // replace namespace only + dockerHost = strings.Replace(c.job.Properties.DockerHost, config.Namespace(), KoderoverAgentNamespace, 1) + } else { + // add namespace + dockerHost = strings.Replace(c.job.Properties.DockerHost, replaceDindServer, replaceDindServer+"."+KoderoverAgentNamespace, 1) + } + } else if c.job.Properties.ClusterID == "" || c.job.Properties.ClusterID == setting.LocalClusterID { + if !strings.Contains(c.job.Properties.DockerHost, config.Namespace()) { + // add namespace + dockerHost = strings.Replace(c.job.Properties.DockerHost, replaceDindServer, replaceDindServer+"."+config.Namespace(), 1) + } + } + + c.job.Properties.DockerHost = dockerHost + + jobCtxBytes, err := yaml.Marshal(BuildJobExcutorContext(c.job, c.workflowCtx, c.logger)) + if err != nil { + msg := fmt.Sprintf("cannot Jobexcutor.Context data: %v", err) + c.logger.Error(msg) + c.job.Status = config.StatusFailed + c.job.Error = msg + return + } + + jobLabel := &JobLabel{ + WorkflowName: c.workflowCtx.WorkflowName, + TaskID: c.workflowCtx.TaskID, + JobType: string(c.job.JobType), + JobName: c.job.Name, + } + if err := ensureDeleteConfigMap(c.job.Properties.Namespace, jobLabel, c.kubeclient); err != nil { + c.logger.Error(err) + c.job.Status = config.StatusFailed + c.job.Error = err.Error() + return + } + + if err := createJobConfigMap( + c.job.Properties.Namespace, c.jobName, jobLabel, string(jobCtxBytes), c.kubeclient); err != nil { + msg := fmt.Sprintf("createJobConfigMap error: %v", err) + c.logger.Error(msg) + c.job.Status = config.StatusFailed + c.job.Error = msg + return + } + + c.logger.Infof("succeed to create cm for job %s", c.jobName) + + // TODO: do not use default image + jobImage := getBaseImage(c.job.Properties.BuildOS, c.job.Properties.ImageFrom) + // jobImage := "koderover.tencentcloudcr.com/test/job-excutor:guoyu-test2" + // jobImage := getReaperImage(config.ReaperImage(), c.job.Properties.BuildOS) + + //Resource request default value is LOW + job, err := buildJob(c.job.JobType, jobImage, c.jobName, c.job.Properties.ClusterID, c.job.Properties.Namespace, c.job.Properties.ResourceRequest, c.job.Properties.ResReqSpec, c.job, c.workflowCtx, nil) + if err != nil { + msg := fmt.Sprintf("create job context error: %v", err) + c.logger.Error(msg) + c.job.Status = config.StatusFailed + c.job.Error = msg + return + } + + job.Namespace = c.job.Properties.Namespace + + if err := ensureDeleteJob(c.job.Properties.Namespace, jobLabel, c.kubeclient); err != nil { + msg := fmt.Sprintf("delete job error: %v", err) + c.logger.Error(msg) + c.job.Status = config.StatusFailed + c.job.Error = msg + return + } + + // 将集成到KodeRover的私有镜像仓库的访问权限设置到namespace中 + // if err := createOrUpdateRegistrySecrets(p.KubeNamespace, pipelineTask.ConfigPayload.RegistryID, p.Task.Registries, p.kubeClient); err != nil { + // msg := fmt.Sprintf("create secret error: %v", err) + // p.Log.Error(msg) + // p.Task.TaskStatus = config.StatusFailed + // p.Task.Error = msg + // p.SetBuildStatusCompleted(config.StatusFailed) + // return + // } + if err := updater.CreateJob(job, c.kubeclient); err != nil { + msg := fmt.Sprintf("create job error: %v", err) + c.logger.Error(msg) + c.job.Status = config.StatusFailed + c.job.Error = msg + return + } + c.logger.Infof("succeed to create job %s", c.jobName) +} + +func (c *FreestyleJobCtl) Wait(ctx context.Context) { + status := waitJobEndWithFile(ctx, int(c.job.Properties.Timeout), c.job.Properties.Namespace, c.jobName, true, c.kubeclient, c.clientset, c.restConfig, c.logger) + c.job.Status = status +} + +func (c *FreestyleJobCtl) Complete(ctx context.Context) { + jobLabel := &JobLabel{ + WorkflowName: c.workflowCtx.WorkflowName, + TaskID: c.workflowCtx.TaskID, + JobType: string(c.job.JobType), + JobName: c.job.Name, + } + + // 清理用户取消和超时的任务 + defer func() { + go func() { + if err := ensureDeleteJob(c.job.Properties.Namespace, jobLabel, c.kubeclient); err != nil { + c.logger.Error(err) + } + if err := ensureDeleteConfigMap(c.job.Properties.Namespace, jobLabel, c.kubeclient); err != nil { + c.logger.Error(err) + } + }() + }() + + // get job outputs info from pod terminate message. + outputs, err := getJobOutput(c.job.Properties.Namespace, c.job.Name, jobLabel, c.kubeclient) + if err != nil { + c.logger.Error(err) + c.job.Error = err.Error() + } + + // write jobs output info to globalcontext so other job can use like this $(jobName.outputName) + for _, output := range outputs { + c.workflowCtx.GlobalContextSet(strings.Join([]string{c.job.Name, output.Name}, "."), output.Value) + } + + if err := saveContainerLog(c.job, c.workflowCtx.WorkflowName, c.workflowCtx.TaskID, jobLabel, c.kubeclient); err != nil { + c.logger.Error(err) + c.job.Error = err.Error() + return + } +} + +func BuildJobExcutorContext(job *commonmodels.JobTask, workflowCtx *commonmodels.WorkflowTaskCtx, logger *zap.SugaredLogger) *JobContext { + var envVars, secretEnvVars []string + for _, env := range job.Properties.Envs { + if env.IsCredential { + secretEnvVars = append(secretEnvVars, strings.Join([]string{env.Key, env.Value}, "=")) + continue + } + envVars = append(envVars, strings.Join([]string{env.Key, env.Value}, "=")) + } + + outputs := []string{} + for _, output := range job.Outputs { + outputs = append(outputs, output.Name) + } + + return &JobContext{ + Name: job.Name, + Envs: envVars, + SecretEnvs: secretEnvVars, + WorkflowName: workflowCtx.WorkflowName, + Workspace: workflowCtx.Workspace, + TaskID: workflowCtx.TaskID, + Outputs: outputs, + Steps: job.Steps, + Paths: job.Properties.Paths, + } +} diff --git a/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job_plugin.go b/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job_plugin.go new file mode 100644 index 0000000000000000000000000000000000000000..983de4260303fb60af11d00958b3b0301ef031dd --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job_plugin.go @@ -0,0 +1,157 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package jobcontroller + +import ( + "context" + "fmt" + "strings" + "time" + + "go.uber.org/zap" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + crClient "sigs.k8s.io/controller-runtime/pkg/client" + + zadigconfig "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/setting" + krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client" + "github.com/koderover/zadig/pkg/tool/kube/updater" +) + +type PluginJobCtl struct { + job *commonmodels.JobTask + jobName string + workflowCtx *commonmodels.WorkflowTaskCtx + logger *zap.SugaredLogger + kubeclient crClient.Client + clientset kubernetes.Interface + restConfig *rest.Config + ack func() +} + +func NewPluginsJobCtl(job *commonmodels.JobTask, workflowCtx *commonmodels.WorkflowTaskCtx, ack func(), logger *zap.SugaredLogger) *PluginJobCtl { + return &PluginJobCtl{ + job: job, + workflowCtx: workflowCtx, + logger: logger, + ack: ack, + jobName: getJobName(workflowCtx.WorkflowName, workflowCtx.TaskID), + } +} + +func (c *PluginJobCtl) Run(ctx context.Context) { + // get kube client + hubServerAddr := config.HubServerAddress() + switch c.job.Properties.ClusterID { + case setting.LocalClusterID: + c.job.Properties.Namespace = zadigconfig.Namespace() + c.kubeclient = krkubeclient.Client() + c.clientset = krkubeclient.Clientset() + c.restConfig = krkubeclient.RESTConfig() + default: + c.job.Properties.Namespace = setting.AttachedClusterNamespace + + crClient, clientset, restConfig, err := GetK8sClients(hubServerAddr, c.job.Properties.ClusterID) + if err != nil { + c.job.Status = config.StatusFailed + c.job.Error = err.Error() + c.job.EndTime = time.Now().Unix() + return + } + c.kubeclient = crClient + c.clientset = clientset + c.restConfig = restConfig + } + + jobLabel := &JobLabel{ + WorkflowName: c.workflowCtx.WorkflowName, + TaskID: c.workflowCtx.TaskID, + JobType: string(c.job.JobType), + JobName: c.job.Name, + } + job, err := buildPlainJob(c.jobName, c.job.Properties.ResourceRequest, c.job.Properties.ResReqSpec, c.job, c.workflowCtx) + if err != nil { + msg := fmt.Sprintf("create job context error: %v", err) + c.logger.Error(msg) + c.job.Status = config.StatusFailed + c.job.Error = msg + return + } + + job.Namespace = c.job.Properties.Namespace + + if err := ensureDeleteJob(c.job.Properties.Namespace, jobLabel, c.kubeclient); err != nil { + msg := fmt.Sprintf("delete job error: %v", err) + c.logger.Error(msg) + c.job.Status = config.StatusFailed + c.job.Error = msg + return + } + if err := updater.CreateJob(job, c.kubeclient); err != nil { + msg := fmt.Sprintf("create job error: %v", err) + c.logger.Error(msg) + c.job.Status = config.StatusFailed + c.job.Error = msg + return + } + c.logger.Infof("succeed to create job %s", c.jobName) +} + +func (c *PluginJobCtl) Wait(ctx context.Context) { + status := waitPlainJobEnd(ctx, int(c.job.Properties.Timeout), c.job.Properties.Namespace, c.jobName, c.kubeclient, c.logger) + c.job.Status = status + +} + +func (c *PluginJobCtl) Complete(ctx context.Context) { + jobLabel := &JobLabel{ + WorkflowName: c.workflowCtx.WorkflowName, + TaskID: c.workflowCtx.TaskID, + JobType: string(c.job.JobType), + JobName: c.job.Name, + } + + // 清理用户取消和超时的任务 + defer func() { + go func() { + if err := ensureDeleteJob(c.job.Properties.Namespace, jobLabel, c.kubeclient); err != nil { + c.logger.Error(err) + } + }() + }() + + // get job outputs info from pod terminate message. + outputs, err := getJobOutput(c.job.Properties.Namespace, c.job.Name, jobLabel, c.kubeclient) + if err != nil { + c.logger.Error(err) + c.job.Error = err.Error() + } + + // write jobs output info to globalcontext so other job can use like this $(jobName.outputName) + for _, output := range outputs { + c.workflowCtx.GlobalContextSet(strings.Join([]string{c.job.Name, output.Name}, "."), output.Value) + } + + if err := saveContainerLog(c.job, c.workflowCtx.WorkflowName, c.workflowCtx.TaskID, jobLabel, c.kubeclient); err != nil { + c.logger.Error(err) + c.job.Error = err.Error() + return + } +} diff --git a/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/kubernetes.go b/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/kubernetes.go new file mode 100644 index 0000000000000000000000000000000000000000..3f764d723c0bb28c07fe10bcbf74758ed7f62632 --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/kubernetes.go @@ -0,0 +1,803 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package jobcontroller + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "path" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + s3tool "github.com/koderover/zadig/pkg/tool/s3" + "go.uber.org/zap" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + crClient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types/task" + "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" + "github.com/koderover/zadig/pkg/shared/kube/wrapper" + "github.com/koderover/zadig/pkg/tool/kube/containerlog" + "github.com/koderover/zadig/pkg/tool/kube/getter" + "github.com/koderover/zadig/pkg/tool/kube/podexec" + "github.com/koderover/zadig/pkg/tool/kube/updater" + "github.com/koderover/zadig/pkg/tool/log" + commontypes "github.com/koderover/zadig/pkg/types" + "github.com/koderover/zadig/pkg/types/job" + "github.com/koderover/zadig/pkg/util" +) + +const ( + BusyBoxImage = "koderover.tencentcloudcr.com/koderover-public/busybox:latest" + ZadigContextDir = "/zadig/" + ZadigLogFile = ZadigContextDir + "zadig.log" + ZadigLifeCycleFile = ZadigContextDir + "lifecycle" + JobExecutorFile = "http://resource-server/jobexecutor" + ResourceServer = "resource-server" +) + +func GetK8sClients(hubServerAddr, clusterID string) (crClient.Client, kubernetes.Interface, *rest.Config, error) { + controllerRuntimeClient, err := kubeclient.GetKubeClient(hubServerAddr, clusterID) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get controller runtime client: %s", err) + } + + clientset, err := kubeclient.GetKubeClientSet(hubServerAddr, clusterID) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get clientset: %s", err) + } + + restConfig, err := kubeclient.GetRESTConfig(hubServerAddr, clusterID) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get rest config: %s", err) + } + + return controllerRuntimeClient, clientset, restConfig, nil +} + +type JobLabel struct { + WorkflowName string + TaskID int64 + JobName string + JobType string +} + +func ensureDeleteConfigMap(namespace string, jobLabel *JobLabel, kubeClient crClient.Client) error { + ls := getJobLabels(jobLabel) + return updater.DeleteConfigMapsAndWait(namespace, labels.Set(ls).AsSelector(), kubeClient) +} + +func ensureDeleteJob(namespace string, jobLabel *JobLabel, kubeClient crClient.Client) error { + ls := getJobLabels(jobLabel) + return updater.DeleteJobsAndWait(namespace, labels.Set(ls).AsSelector(), kubeClient) +} + +// getJobLabels get labels k-v map from JobLabel struct +func getJobLabels(jobLabel *JobLabel) map[string]string { + retMap := map[string]string{ + setting.JobLabelTaskKey: fmt.Sprintf("%s-%d", strings.ToLower(jobLabel.WorkflowName), jobLabel.TaskID), + setting.JobLabelNameKey: strings.Replace(jobLabel.JobName, "_", "-", -1), + setting.JobLabelSTypeKey: strings.Replace(jobLabel.JobType, "_", "-", -1), + } + // no need to add labels with empty value to a job + for k, v := range retMap { + if len(v) == 0 { + delete(retMap, k) + } + } + return retMap +} + +func createJobConfigMap(namespace, jobName string, jobLabel *JobLabel, jobCtx string, kubeClient crClient.Client) error { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: jobName, + Namespace: namespace, + Labels: getJobLabels(jobLabel), + }, + Data: map[string]string{ + "job-config.xml": jobCtx, + }, + } + + return updater.CreateConfigMap(cm, kubeClient) +} + +func getBaseImage(buildOS, imageFrom string) string { + // for built-in image, reaperImage and buildOs can generate a complete image + // reaperImage: koderover.tencentcloudcr.com/koderover-public/build-base:${BuildOS}-amd64 + // buildOS: focal xenial bionic + jobImage := strings.ReplaceAll(config.ReaperImage(), "${BuildOS}", buildOS) + // for custom image, buildOS represents the exact custom image + if imageFrom == setting.ImageFromCustom { + jobImage = buildOS + } + return jobImage +} + +func buildPlainJob(jobName string, resReq setting.Request, resReqSpec setting.RequestSpec, jobTask *commonmodels.JobTask, workflowCtx *commonmodels.WorkflowTaskCtx) (*batchv1.Job, error) { + collectJobOutput := `OLD_IFS=$IFS +export IFS="," +files='%s' +outputs='%s' +file_arr=($files) +output_arr=($outputs) +IFS="$OLD_IFS" +result="{" +for i in ${!file_arr[@]}; +do + file_value=$(cat ${file_arr[$i]}) + output_value=${output_arr[$i]} + result="$result\"$output_value\":\"$file_value\"," +done +result=$(sed 's/,$/}/' <<< $result) +echo $result > %s +` + files := []string{} + outputs := []string{} + for _, output := range jobTask.Outputs { + outputFile := path.Join(job.JobOutputDir, output.Name) + files = append(files, outputFile) + outputs = append(outputs, output.Name) + } + collectJobOutputCommand := fmt.Sprintf(collectJobOutput, strings.Join(files, ","), strings.Join(outputs, ","), job.JobTerminationFile) + + labels := getJobLabels(&JobLabel{ + WorkflowName: workflowCtx.WorkflowName, + TaskID: workflowCtx.TaskID, + JobType: string(jobTask.JobType), + JobName: jobTask.Name, + }) + + ImagePullSecrets := []corev1.LocalObjectReference{ + { + Name: setting.DefaultImagePullSecret, + }, + } + envs := []corev1.EnvVar{} + for _, env := range jobTask.Plugin.Envs { + envs = append(envs, corev1.EnvVar{Name: env.Name, Value: env.Value}) + } + + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: jobName, + Labels: labels, + }, + Spec: batchv1.JobSpec{ + Completions: int32Ptr(1), + Parallelism: int32Ptr(1), + BackoffLimit: int32Ptr(0), + // in case finished zombie job not cleaned up by zadig + TTLSecondsAfterFinished: int32Ptr(3600), + // in case zombie job never stop + ActiveDeadlineSeconds: int64Ptr(jobTask.Properties.Timeout*60 + 3600), + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + ImagePullSecrets: ImagePullSecrets, + Containers: []corev1.Container{ + { + ImagePullPolicy: corev1.PullAlways, + Name: jobTask.Name, + Image: jobTask.Plugin.Image, + Args: jobTask.Plugin.Args, + Command: jobTask.Plugin.Cmds, + Lifecycle: &corev1.Lifecycle{ + PostStart: &corev1.Handler{ + Exec: &corev1.ExecAction{ + Command: []string{"/bin/sh", "-c", fmt.Sprintf("mkdir -p %s", job.JobOutputDir)}, + }, + }, + PreStop: &corev1.Handler{ + Exec: &corev1.ExecAction{ + Command: []string{"/bin/sh", "-c", collectJobOutputCommand}, + }, + }, + }, + Env: envs, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "zadig-context", + MountPath: ZadigContextDir, + }, + }, + Resources: getResourceRequirements(resReq, resReqSpec), + + TerminationMessagePolicy: corev1.TerminationMessageReadFile, + TerminationMessagePath: job.JobTerminationFile, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "zadig-context", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + }, + }, + }, + } + return job, nil +} + +func buildJob(jobType, jobImage, jobName, clusterID, currentNamespace string, resReq setting.Request, resReqSpec setting.RequestSpec, jobTask *commonmodels.JobTask, workflowCtx *commonmodels.WorkflowTaskCtx, registries []*task.RegistryNamespace) (*batchv1.Job, error) { + // tailLogCommandTemplate := `tail -f %s & + // while [ -f %s ]; + // do + // sleep 1s; + // done; + // ` + // tailLogCommand := fmt.Sprintf(tailLogCommandTemplate, ZadigLogFile, ZadigLifeCycleFile) + + var ( + jobExecutorBootingScript string + jobExecutorBinaryFile = JobExecutorFile + ) + // not local cluster + if clusterID != "" && clusterID != setting.LocalClusterID { + jobExecutorBinaryFile = strings.Replace(jobExecutorBinaryFile, ResourceServer, ResourceServer+".koderover-agent", -1) + } else { + jobExecutorBinaryFile = strings.Replace(jobExecutorBinaryFile, ResourceServer, ResourceServer+"."+currentNamespace, -1) + } + + jobExecutorBootingScript = fmt.Sprintf("curl -m 10 --retry-delay 3 --retry 3 -sSL %s -o reaper && chmod +x reaper && mv reaper /usr/local/bin && /usr/local/bin/reaper", jobExecutorBinaryFile) + + labels := getJobLabels(&JobLabel{ + WorkflowName: workflowCtx.WorkflowName, + TaskID: workflowCtx.TaskID, + JobType: string(jobType), + JobName: jobTask.Name, + }) + + // 引用集成到系统中的私有镜像仓库的访问权限 + ImagePullSecrets := []corev1.LocalObjectReference{ + { + Name: setting.DefaultImagePullSecret, + }, + } + // for _, reg := range registries { + // secretName, err := genRegistrySecretName(reg) + // if err != nil { + // return nil, fmt.Errorf("failed to generate registry secret name: %s", err) + // } + + // secret := corev1.LocalObjectReference{ + // Name: secretName, + // } + // ImagePullSecrets = append(ImagePullSecrets, secret) + // } + + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: jobName, + Labels: labels, + }, + Spec: batchv1.JobSpec{ + Completions: int32Ptr(1), + Parallelism: int32Ptr(1), + BackoffLimit: int32Ptr(0), + // in case finished zombie job not cleaned up by zadig + TTLSecondsAfterFinished: int32Ptr(3600), + // in case zombie job never stop + ActiveDeadlineSeconds: int64Ptr(jobTask.Properties.Timeout*60 + 3600), + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + ImagePullSecrets: ImagePullSecrets, + // InitContainers: []corev1.Container{ + // { + // ImagePullPolicy: corev1.PullIfNotPresent, + // Name: "init-log-file", + // Image: BusyBoxImage, + // VolumeMounts: getVolumeMounts(workflowCtx.ConfigMapMountDir), + // Command: []string{"/bin/sh", "-c", fmt.Sprintf("touch %s %s", ZadigLogFile, ZadigLifeCycleFile)}, + // }, + // }, + Containers: []corev1.Container{ + { + ImagePullPolicy: corev1.PullAlways, + Name: jobTask.Name, + Image: jobImage, + Command: []string{"/bin/sh", "-c"}, + Args: []string{jobExecutorBootingScript}, + // Command: []string{"/bin/sh", "-c", "jobexecutor"}, + // Lifecycle: &corev1.Lifecycle{ + // PreStop: &corev1.Handler{ + // Exec: &corev1.ExecAction{ + // Command: []string{"/bin/sh", "-c", fmt.Sprintf("rm %s", ZadigLifeCycleFile)}, + // }, + // }, + // }, + Env: []corev1.EnvVar{ + { + Name: "JOB_CONFIG_FILE", + Value: path.Join(workflowCtx.ConfigMapMountDir, "job-config.xml"), + }, + // 连接对应wd上的dockerdeamon + { + Name: "DOCKER_HOST", + Value: jobTask.Properties.DockerHost, + }, + }, + VolumeMounts: getVolumeMounts(workflowCtx.ConfigMapMountDir), + Resources: getResourceRequirements(resReq, resReqSpec), + + TerminationMessagePolicy: corev1.TerminationMessageReadFile, + TerminationMessagePath: job.JobTerminationFile, + }, + // { + // ImagePullPolicy: corev1.PullIfNotPresent, + // Name: "log", + // Image: BusyBoxImage, + // VolumeMounts: getVolumeMounts(workflowCtx.ConfigMapMountDir), + // Command: []string{"/bin/sh", "-c"}, + // Args: []string{tailLogCommand}, + // Lifecycle: &corev1.Lifecycle{}, + // }, + }, + Volumes: getVolumes(jobName), + }, + }, + }, + } + + if jobTask.Properties.CacheEnable && jobTask.Properties.Cache.MediumType == commontypes.NFSMedium { + volumeName := "build-cache" + job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: volumeName, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: jobTask.Properties.Cache.NFSProperties.PVC, + }, + }, + }) + + mountPath := jobTask.Properties.CacheUserDir + if jobTask.Properties.CacheDirType == commontypes.WorkspaceCacheDir { + mountPath = workflowCtx.Workspace + } + + job.Spec.Template.Spec.Containers[0].VolumeMounts = append(job.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ + Name: volumeName, + MountPath: mountPath, + SubPath: jobTask.Properties.Cache.NFSProperties.Subpath, + }) + } + + // if affinity := addNodeAffinity(clusterID, pipelineTask.ConfigPayload.K8SClusters); affinity != nil { + // job.Spec.Template.Spec.Affinity = affinity + // } + + return job, nil +} + +func getVolumeMounts(configMapMountDir string) []corev1.VolumeMount { + resp := make([]corev1.VolumeMount, 0) + + resp = append(resp, corev1.VolumeMount{ + Name: "job-config", + MountPath: configMapMountDir, + }) + resp = append(resp, corev1.VolumeMount{ + Name: "zadig-context", + MountPath: ZadigContextDir, + }) + + return resp +} + +func getVolumes(jobName string) []corev1.Volume { + resp := make([]corev1.Volume, 0) + resp = append(resp, corev1.Volume{ + Name: "job-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: jobName, + }, + }, + }, + }) + resp = append(resp, corev1.Volume{ + Name: "zadig-context", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }) + return resp +} + +func getResourceRequirements(resReq setting.Request, resReqSpec setting.RequestSpec) corev1.ResourceRequirements { + + switch resReq { + case setting.HighRequest: + return generateResourceRequirements(setting.HighRequest, setting.HighRequestSpec) + + case setting.MediumRequest: + return generateResourceRequirements(setting.MediumRequest, setting.MediumRequestSpec) + + case setting.LowRequest: + return generateResourceRequirements(setting.LowRequest, setting.LowRequestSpec) + + case setting.MinRequest: + return generateResourceRequirements(setting.MinRequest, setting.MinRequestSpec) + + case setting.DefineRequest: + return generateResourceRequirements(resReq, resReqSpec) + + default: + return generateResourceRequirements(setting.DefaultRequest, setting.DefaultRequestSpec) + } +} + +//generateResourceRequirements +//cpu Request:Limit=1:4 +//memory default Request:Limit=1:4 ; if memoryLimit>= 8Gi,Request:Limit=1:8 +func generateResourceRequirements(req setting.Request, reqSpec setting.RequestSpec) corev1.ResourceRequirements { + + if req != setting.DefineRequest { + return corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(strconv.Itoa(reqSpec.CpuLimit) + setting.CpuUintM), + corev1.ResourceMemory: resource.MustParse(strconv.Itoa(reqSpec.MemoryLimit) + setting.MemoryUintMi), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(strconv.Itoa(reqSpec.CpuReq) + setting.CpuUintM), + corev1.ResourceMemory: resource.MustParse(strconv.Itoa(reqSpec.MemoryReq) + setting.MemoryUintMi), + }, + } + } + + cpuReqInt := reqSpec.CpuLimit / 4 + if cpuReqInt < 1 { + cpuReqInt = 1 + } + memoryReqInt := reqSpec.MemoryLimit / 4 + if memoryReqInt >= 2*1024 { + memoryReqInt = memoryReqInt / 2 + } + if memoryReqInt < 1 { + memoryReqInt = 1 + } + + return corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(strconv.Itoa(reqSpec.CpuLimit) + setting.CpuUintM), + corev1.ResourceMemory: resource.MustParse(strconv.Itoa(reqSpec.MemoryLimit) + setting.MemoryUintMi), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(strconv.Itoa(cpuReqInt) + setting.CpuUintM), + corev1.ResourceMemory: resource.MustParse(strconv.Itoa(memoryReqInt) + setting.MemoryUintMi), + }, + } +} + +func int32Ptr(i int32) *int32 { return &i } +func int64Ptr(i int64) *int64 { return &i } + +func waitPlainJobEnd(ctx context.Context, taskTimeout int, namespace, jobName string, kubeClient crClient.Client, xl *zap.SugaredLogger) config.Status { + xl.Infof("wait job to start: %s/%s", namespace, jobName) + timeout := time.After(time.Duration(taskTimeout) * time.Minute) + podTimeout := time.After(120 * time.Second) + + xl.Infof("Timeout of preparing Pod: %s. Timeout of running task: %s.", 120*time.Second, time.Duration(taskTimeout)*time.Minute) + + var started bool + for { + select { + case <-ctx.Done(): + return config.StatusCancelled + + case <-podTimeout: + return config.StatusTimeout + default: + job, _, err := getter.GetJob(namespace, jobName, kubeClient) + if err != nil { + xl.Errorf("get job failed, namespace:%s, jobName:%s, err:%v", namespace, jobName, err) + } + if job != nil { + started = job.Status.Active > 0 + } + } + if started { + break + } + + time.Sleep(time.Second) + } + // wait for the job to end. + xl.Infof("wait job to end: %s %s", namespace, jobName) + for { + select { + case <-ctx.Done(): + return config.StatusCancelled + + case <-timeout: + return config.StatusTimeout + + default: + job, found, err := getter.GetJob(namespace, jobName, kubeClient) + if err != nil || !found { + xl.Errorf("failed to get pod with label job-name=%s %v", jobName, err) + return config.StatusFailed + } + + if job.Status.Succeeded != 0 { + return config.StatusPassed + } + if job.Status.Failed != 0 { + return config.StatusFailed + } + } + + time.Sleep(time.Second * 1) + } + +} + +func waitJobEndWithFile(ctx context.Context, taskTimeout int, namespace, jobName string, checkFile bool, kubeClient crClient.Client, clientset kubernetes.Interface, restConfig *rest.Config, xl *zap.SugaredLogger) (status config.Status) { + xl.Infof("wait job to start: %s/%s", namespace, jobName) + timeout := time.After(time.Duration(taskTimeout) * time.Minute) + podTimeout := time.After(120 * time.Second) + + xl.Infof("Timeout of preparing Pod: %s. Timeout of running task: %s.", 120*time.Second, time.Duration(taskTimeout)*time.Minute) + + var started bool + for { + select { + case <-ctx.Done(): + return config.StatusCancelled + + case <-podTimeout: + return config.StatusTimeout + default: + job, _, err := getter.GetJob(namespace, jobName, kubeClient) + if err != nil { + xl.Errorf("get job failed, namespace:%s, jobName:%s, err:%v", namespace, jobName, err) + } + if job != nil { + started = job.Status.Active > 0 + } + } + if started { + break + } + + time.Sleep(time.Second) + } + + // 等待job 运行结束 + xl.Infof("wait job to end: %s %s", namespace, jobName) + for { + select { + case <-ctx.Done(): + return config.StatusCancelled + + case <-timeout: + return config.StatusTimeout + + default: + job, found, err := getter.GetJob(namespace, jobName, kubeClient) + if err != nil || !found { + xl.Errorf("failed to get pod with label job-name=%s %v", jobName, err) + return config.StatusFailed + } + // pod is still running + if job.Status.Active != 0 { + if !checkFile { + // break only break the select{}, not the outside for{} + break + } + + pods, err := getter.ListPods(namespace, labels.Set{"job-name": jobName}.AsSelector(), kubeClient) + if err != nil { + xl.Errorf("failed to find pod with label job-name=%s %v", jobName, err) + return config.StatusFailed + } + var done, exists bool + var jobStatus commontypes.JobStatus + + for _, pod := range pods { + ipod := wrapper.Pod(pod) + if ipod.Pending() { + continue + } + if ipod.Failed() { + return config.StatusFailed + } + if !ipod.Finished() { + jobStatus, exists, err = checkDogFoodExistsInContainer(clientset, restConfig, namespace, ipod.Name, ipod.ContainerNames()[0]) + if err != nil { + // Note: + // Currently, this error indicates "the target Pod cannot be accessed" or "the target Pod can be accessed, but the dog food file does not exist". + // In these two scenarios, `Info` is used to print logs because they are not business semantic exceptions. + xl.Infof("Result of checking dog food file %s: %s", pods[0].Name, err) + break + } + if !exists { + break + } + } + done = true + } + + if done { + xl.Infof("Dog food is found, stop to wait %s. Job status: %s.", job.Name, jobStatus) + + switch jobStatus { + case commontypes.JobFail: + return config.StatusFailed + default: + return config.StatusPassed + } + } + } else if job.Status.Succeeded != 0 { + return config.StatusPassed + } else { + return config.StatusFailed + } + } + + time.Sleep(time.Second * 1) + } +} + +func getJobOutput(namespace, containerName string, jobLabel *JobLabel, kubeClient crClient.Client) ([]*job.JobOutput, error) { + resp := []*job.JobOutput{} + ls := getJobLabels(jobLabel) + pods, err := getter.ListPods(namespace, labels.Set(ls).AsSelector(), kubeClient) + if err != nil { + return resp, err + } + for _, pod := range pods { + ipod := wrapper.Pod(pod) + // only collect succeeed job outputs. + if !ipod.Succeeded() { + return resp, nil + } + for _, containerStatus := range pod.Status.ContainerStatuses { + if containerStatus.Name != ls[containerName] { + continue + } + if containerStatus.State.Terminated != nil && len(containerStatus.State.Terminated.Message) != 0 { + if err := json.Unmarshal([]byte(containerStatus.State.Terminated.Message), &resp); err != nil { + return resp, err + } + return resp, nil + } + } + } + return resp, nil +} + +func saveContainerLog(job *commonmodels.JobTask, workflowName string, taskID int64, jobLabel *JobLabel, kubeClient crClient.Client) error { + selector := labels.Set(getJobLabels(jobLabel)).AsSelector() + pods, err := getter.ListPods(job.Properties.Namespace, selector, kubeClient) + if err != nil { + return err + } + + if len(pods) < 1 { + return fmt.Errorf("no pod found with selector: %s", selector) + } + + if len(pods[0].Status.ContainerStatuses) < 1 { + return fmt.Errorf("no cotainer statuses : %s", selector) + } + + buf := new(bytes.Buffer) + // 默认取第一个build job的第一个pod的第一个container的日志 + sort.SliceStable(pods, func(i, j int) bool { + return pods[i].CreationTimestamp.Before(&pods[j].CreationTimestamp) + }) + + clientSet, err := kubeclient.GetClientset(config.HubServerAddress(), job.Properties.ClusterID) + if err != nil { + log.Errorf("saveContainerLog, get client set error: %s", err) + return err + } + + if err := containerlog.GetContainerLogs(job.Properties.Namespace, pods[0].Name, pods[0].Spec.Containers[0].Name, false, int64(0), buf, clientSet); err != nil { + return fmt.Errorf("failed to get container logs: %s", err) + } + + store, err := commonrepo.NewS3StorageColl().FindDefault() + if err != nil { + return fmt.Errorf("failed to get default s3 storage: %s", err) + } + + if tempFileName, err := util.GenerateTmpFile(); err == nil { + defer func() { + _ = os.Remove(tempFileName) + }() + if err = saveFile(buf, tempFileName); err == nil { + + if store.Subfolder != "" { + store.Subfolder = fmt.Sprintf("%s/%s/%d/%s", store.Subfolder, strings.ToLower(workflowName), taskID, "log") + } else { + store.Subfolder = fmt.Sprintf("%s/%d/%s", strings.ToLower(workflowName), taskID, "log") + } + forcedPathStyle := true + if store.Provider == setting.ProviderSourceAli { + forcedPathStyle = false + } + s3client, err := s3tool.NewClient(store.Endpoint, store.Ak, store.Sk, store.Insecure, forcedPathStyle) + if err != nil { + return fmt.Errorf("saveContainerLog s3 create client error: %v", err) + } + fileName := strings.Replace(strings.ToLower(job.Name), "_", "-", -1) + objectKey := GetObjectPath(store.Subfolder, fileName+".log") + if err = s3client.Upload( + store.Bucket, + tempFileName, + objectKey, + ); err != nil { + return fmt.Errorf("saveContainerLog s3 Upload error: %v", err) + } + } else { + return fmt.Errorf("saveContainerLog saveFile error: %v", err) + } + } else { + return fmt.Errorf("saveContainerLog GenerateTmpFile error: %v", err) + } + return nil +} + +func GetObjectPath(subFolder, name string) string { + // target should not be started with / + if subFolder != "" { + return strings.TrimLeft(filepath.Join(subFolder, name), "/") + } + + return strings.TrimLeft(name, "/") +} + +func checkDogFoodExistsInContainer(clientset kubernetes.Interface, restConfig *rest.Config, namespace, pod, container string) (commontypes.JobStatus, bool, error) { + stdout, _, success, err := podexec.KubeExec(clientset, restConfig, podexec.ExecOptions{ + Command: []string{"/bin/sh", "-c", fmt.Sprintf("test -f %[1]s && cat %[1]s", setting.DogFood)}, + Namespace: namespace, + PodName: pod, + ContainerName: container, + }) + + return commontypes.JobStatus(stdout), success, err +} diff --git a/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/types.go b/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/types.go new file mode 100644 index 0000000000000000000000000000000000000000..f5b584b5565882be54b243e1a56651247e7eb360 --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/types.go @@ -0,0 +1,55 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package jobcontroller + +import ( + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" +) + +type JobContext struct { + Name string `yaml:"name"` + // Workspace 容器工作目录 [必填] + Workspace string `yaml:"workspace"` + Proxy *Proxy `yaml:"proxy"` + // Envs 用户注入环境变量, 包括安装脚本环境变量 [optional] + Envs EnvVar `yaml:"envs"` + // SecretEnvs 用户注入敏感信息环境变量, value不能在stdout stderr中输出 [optional] + SecretEnvs EnvVar `yaml:"secret_envs"` + // WorkflowName + WorkflowName string `yaml:"workflow_name"` + // TaskID + TaskID int64 `yaml:"task_id"` + // Paths 执行脚本Path + Paths string `yaml:"paths"` + + Steps []*commonmodels.StepTask `yaml:"steps"` + Outputs []string `yaml:"outputs"` +} + +type EnvVar []string + +// Proxy 翻墙配置信息 +type Proxy struct { + Type string `yaml:"type"` + Address string `yaml:"address"` + Port int `yaml:"port"` + NeedPassword bool `yaml:"need_password"` + Username string `yaml:"username"` + Password string `yaml:"password"` + EnableRepoProxy bool `yaml:"enable_repo_proxy"` + EnableApplicationProxy bool `yaml:"enable_application_proxy"` +} diff --git a/pkg/microservice/aslan/core/common/service/workflowcontroller/queue.go b/pkg/microservice/aslan/core/common/service/workflowcontroller/queue.go new file mode 100644 index 0000000000000000000000000000000000000000..8da775356f38e8bd53b2fe427ea7d951e391c49a --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/workflowcontroller/queue.go @@ -0,0 +1,306 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package workflowcontroller + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/log" +) + +func RunningTasks() []*commonmodels.WorkflowQueue { + tasks := make([]*commonmodels.WorkflowQueue, 0) + queueTasks, err := commonrepo.NewWorkflowQueueColl().List(&commonrepo.ListWorfklowQueueOption{}) + if err != nil { + log.Errorf("list queue worklfow task failed, err:%v", err) + return tasks + } + for _, t := range queueTasks { + // task状态为TaskQueued说明task已经被send到nsq,wd已经开始处理但是没有返回ack + if t.Status == config.StatusRunning { + tasks = append(tasks, t) + } + } + return tasks +} + +func PendingTasks() []*commonmodels.WorkflowQueue { + tasks := make([]*commonmodels.WorkflowQueue, 0) + queueTasks, err := commonrepo.NewWorkflowQueueColl().List(&commonrepo.ListWorfklowQueueOption{}) + if err != nil { + log.Errorf("list queue workflow task failed, err:%v", err) + return tasks + } + for _, t := range queueTasks { + if t.Status == config.StatusWaiting || t.Status == config.StatusBlocked || t.Status == config.StatusQueued { + tasks = append(tasks, t) + } + } + return tasks +} + +type CancelMessage struct { + Revoker string `json:"revoker"` + PipelineName string `json:"pipeline_name"` + TaskID int64 `json:"task_id"` + ReqID string `json:"req_id"` +} + +// CreateTask 接受create task请求, 保存task到数据库, 发送task到queue +func CreateTask(t *commonmodels.WorkflowTask) error { + t.Status = config.StatusWaiting + if _, err := commonrepo.NewworkflowTaskv4Coll().Create(t); err != nil { + log.Errorf("create workflow task v4 error: %v", err) + return err + } + return Push(t) +} + +func UpdateTask(t *commonmodels.WorkflowTask) error { + t.Status = config.StatusWaiting + if err := commonrepo.NewworkflowTaskv4Coll().Update(t.ID.Hex(), t); err != nil { + log.Errorf("create workflow task v4 error: %v", err) + return err + } + return Push(t) +} + +func Push(t *commonmodels.WorkflowTask) error { + if t == nil { + return errors.New("nil task") + } + + if !t.MultiRun { + opt := &commonrepo.ListWorfklowQueueOption{ + WorkflowName: t.WorkflowName, + } + tasks, err := commonrepo.NewWorkflowQueueColl().List(opt) + if err == nil && len(tasks) > 0 { + log.Infof("blocked task recevied: %v %v %v", t.CreateTime, t.TaskID, t.WorkflowName) + t.Status = config.StatusBlocked + } + } + + if err := commonrepo.NewWorkflowQueueColl().Create(ConvertTaskToQueue(t)); err != nil { + log.Errorf("workflowTaskV4.Create error: %v", err) + return err + } + return nil +} + +func InitWorkflowController() { + InitQueue() + go WorfklowTaskSender() +} + +func InitQueue() error { + log := log.SugaredLogger() + + // 从数据库查找未完成的任务 + // status = created, running + tasks, err := commonrepo.NewworkflowTaskv4Coll().InCompletedTasks() + if err != nil { + log.Errorf("find [InCompletedTasks] error: %v", err) + return err + } + + for _, task := range tasks { + // 如果 Queue 重新初始化, 取消所有 running tasks + if err := CancelWorkflowTask(setting.DefaultTaskRevoker, task.WorkflowName, task.TaskID, log); err != nil { + log.Errorf("[CancelRunningTask] error: %v", err) + continue + } + } + return nil +} + +// WorfklowTaskSender 监控warpdrive空闲情况, 如果有空闲, 则发现下一个waiting task给warpdrive +// 并将task状态设置为queued +func WorfklowTaskSender() { + for { + time.Sleep(time.Second * 3) + + sysSetting, err := commonrepo.NewSystemSettingColl().Get() + if err != nil { + log.Errorf("get system stettings error: %v", err) + } + //c.checkAgents() + if !hasAgentAvaiable(int(sysSetting.WorkflowConcurrency)) { + continue + } + t, err := NextWaitingTask() + if err != nil { + // no waiting task found + blockTasks, err := BlockedTaskQueue() + if err != nil { + //no blocked task found + continue + } + for _, blockTask := range blockTasks { + if hasAgentAvaiable(int(sysSetting.WorkflowConcurrency)) { + //判断相同的工作流是否正在运行 + if ParallelRunningAndQueuedTasks(blockTask) { + continue + } + // update agent and queue + if err := updateQueueAndRunTask(blockTask, int(sysSetting.BuildConcurrency)); err != nil { + continue + } + } else { + break + } + } + continue + } + // update agent and queue + if err := updateQueueAndRunTask(t, int(sysSetting.BuildConcurrency)); err != nil { + continue + } + + } +} + +func hasAgentAvaiable(workflowConcurrency int) bool { + return len(RunningAndQueuedTasks()) < int(workflowConcurrency) +} + +func RunningAndQueuedTasks() []*commonmodels.WorkflowQueue { + tasks := make([]*commonmodels.WorkflowQueue, 0) + for _, t := range ListTasks() { + // task状态为TaskQueued说明task已经被send到nsq,wd已经开始处理但是没有返回ack + if t.Status == config.StatusRunning || t.Status == config.StatusQueued { + tasks = append(tasks, t) + } + } + return tasks +} + +func ListTasks() []*commonmodels.WorkflowQueue { + opt := new(commonrepo.ListWorfklowQueueOption) + queues, err := commonrepo.NewWorkflowQueueColl().List(opt) + if err != nil { + log.Errorf("pqColl.List error: %v", err) + } + return queues +} + +// NextWaitingTask 查询下一个等待的task +func NextWaitingTask() (*commonmodels.WorkflowQueue, error) { + opt := &commonrepo.ListWorfklowQueueOption{ + Status: config.StatusWaiting, + } + + tasks, err := commonrepo.NewWorkflowQueueColl().List(opt) + if err != nil { + return nil, err + } + + for _, t := range tasks { + return t, nil + } + + return nil, errors.New("no waiting task found") +} + +func BlockedTaskQueue() ([]*commonmodels.WorkflowQueue, error) { + opt := &commonrepo.ListWorfklowQueueOption{ + Status: config.StatusBlocked, + } + + queues, err := commonrepo.NewWorkflowQueueColl().List(opt) + if err != nil || len(queues) == 0 { + return nil, errors.New("no blocked task found") + } + + return queues, nil +} + +func ParallelRunningAndQueuedTasks(currentTask *commonmodels.WorkflowQueue) bool { + for _, t := range ListTasks() { + // task状态为TaskQueued说明task已经被send到nsq,wd已经开始处理但是没有返回ack + if t.Status != config.StatusRunning && t.Status != config.StatusQueued { + continue + } + if t.WorkflowName == currentTask.WorkflowName { + return true + } + } + return false +} + +func updateQueueAndRunTask(t *commonmodels.WorkflowQueue, jobConcurrency int) error { + logger := log.SugaredLogger() + // 更新队列状态为TaskQueued + workflowTask, err := commonrepo.NewworkflowTaskv4Coll().Find(t.WorkflowName, t.TaskID) + if err != nil { + logger.Errorf("%s:%d get workflow task error: %v", t.WorkflowName, t.TaskID, err) + return fmt.Errorf("%s:%d get workflow task error: %v", t.WorkflowName, t.TaskID, err) + } + workflowTask.Status = config.StatusQueued + if success := UpdateQueue(workflowTask); !success { + logger.Errorf("%s:%d update t status error", t.WorkflowName, t.TaskID) + return fmt.Errorf("%s:%d update t status error", t.WorkflowName, t.TaskID) + } + ctx := context.Background() + go NewWorkflowController(workflowTask, logger).Run(ctx, jobConcurrency) + return nil +} + +func UpdateQueue(task *commonmodels.WorkflowTask) bool { + if err := commonrepo.NewWorkflowQueueColl().Update(ConvertTaskToQueue(task)); err != nil { + return false + } + return true +} + +func ConvertTaskToQueue(task *commonmodels.WorkflowTask) *commonmodels.WorkflowQueue { + return &commonmodels.WorkflowQueue{ + TaskID: task.TaskID, + WorkflowName: task.WorkflowName, + ProjectName: task.ProjectName, + Status: task.Status, + Stages: cleanStages(task.Stages), + TaskCreator: task.TaskCreator, + TaskRevoker: task.TaskRevoker, + CreateTime: task.CreateTime, + MultiRun: task.MultiRun, + } +} + +func cleanStages(stages []*commonmodels.StageTask) []*commonmodels.StageTask { + resp := []*commonmodels.StageTask{} + data, _ := json.Marshal(stages) + json.Unmarshal(data, &resp) + for _, stage := range resp { + for _, job := range stage.Jobs { + job.Steps = nil + } + } + return resp +} + +func Remove(taskQueue *commonmodels.WorkflowQueue) error { + return commonrepo.NewWorkflowQueueColl().Delete(taskQueue) +} diff --git a/pkg/microservice/aslan/core/common/service/workflowcontroller/stage.go b/pkg/microservice/aslan/core/common/service/workflowcontroller/stage.go new file mode 100644 index 0000000000000000000000000000000000000000..0ff589cfb5f795213e7a31995a9979d5a3e77ee0 --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/workflowcontroller/stage.go @@ -0,0 +1,243 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package workflowcontroller + +import ( + "context" + "fmt" + "sync" + "time" + + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" +) + +type approveMap struct { + m map[string]*approveWithLock + sync.RWMutex +} + +type approveWithLock struct { + approval *commonmodels.Approval + sync.RWMutex +} + +var globalApproveMap approveMap + +func init() { + globalApproveMap.m = make(map[string]*approveWithLock, 0) +} + +type StageCtl interface { + Run(ctx context.Context, concurrency int) +} + +func runStage(ctx context.Context, stage *commonmodels.StageTask, workflowCtx *commonmodels.WorkflowTaskCtx, concurrency int, logger *zap.SugaredLogger, ack func()) { + stage.Status = config.StatusRunning + stage.StartTime = time.Now().Unix() + ack() + logger.Infof("start stage: %s,status: %s", stage.Name, stage.Status) + if err := waitiForApprove(ctx, stage, workflowCtx, ack); err != nil { + stage.Error = err.Error() + stage.EndTime = time.Now().Unix() + logger.Errorf("finish stage: %s,status: %s", stage.Name, stage.Status) + ack() + return + } + defer func() { + updateStageStatus(stage) + stage.EndTime = time.Now().Unix() + logger.Infof("finish stage: %s,status: %s", stage.Name, stage.Status) + ack() + }() + + stageCtl := NewCustomStageCtl(stage, workflowCtx, logger, ack) + + stageCtl.Run(ctx, concurrency) +} + +func RunStages(ctx context.Context, stages []*commonmodels.StageTask, workflowCtx *commonmodels.WorkflowTaskCtx, concurrency int, logger *zap.SugaredLogger, ack func()) { + for _, stage := range stages { + runStage(ctx, stage, workflowCtx, concurrency, logger, ack) + if statusFailed(stage.Status) { + return + } + } +} + +func ApproveStage(workflowName, stageName, userName, userID, comment string, taskID int64, approve bool) error { + approveKey := fmt.Sprintf("%s-%d-%s", workflowName, taskID, stageName) + approveWithL, ok := globalApproveMap.getApproval(approveKey) + if !ok { + return fmt.Errorf("workflow %s ID %d stage %s do not need approve", workflowName, taskID, stageName) + } + return approveWithL.doApproval(userName, userID, comment, approve) +} + +func waitiForApprove(ctx context.Context, stage *commonmodels.StageTask, workflowCtx *commonmodels.WorkflowTaskCtx, ack func()) error { + if stage.Approval == nil { + return nil + } + if !stage.Approval.Enabled { + return nil + } + if stage.Approval.Timeout == 0 { + stage.Approval.Timeout = 60 + } + approveKey := fmt.Sprintf("%s-%d-%s", workflowCtx.WorkflowName, workflowCtx.TaskID, stage.Name) + approveWithL := &approveWithLock{approval: stage.Approval} + globalApproveMap.setApproval(approveKey, approveWithL) + defer func() { + globalApproveMap.deleteApproval(approveKey) + ack() + }() + + timeout := time.After(time.Duration(stage.Approval.Timeout) * time.Minute) + latestApproveCount := 0 + for { + time.Sleep(1 * time.Second) + select { + case <-ctx.Done(): + stage.Status = config.StatusCancelled + return fmt.Errorf("workflow was canceled") + + case <-timeout: + stage.Status = config.StatusCancelled + return fmt.Errorf("workflow timeout") + default: + approved, approveCount, err := approveWithL.isApproval() + if err != nil { + stage.Status = config.StatusReject + return err + } + if approved { + return nil + } + if approveCount > latestApproveCount { + ack() + latestApproveCount = approveCount + } + } + } +} + +func statusFailed(status config.Status) bool { + if status == config.StatusCancelled || status == config.StatusFailed || status == config.StatusTimeout || status == config.StatusReject { + return true + } + return false +} + +func updateStageStatus(stage *commonmodels.StageTask) { + statusMap := map[config.Status]int{ + config.StatusCancelled: 4, + config.StatusTimeout: 3, + config.StatusFailed: 2, + config.StatusPassed: 1, + config.StatusSkipped: 0, + } + + // 初始化stageStatus为创建状态 + stageStatus := config.StatusRunning + + jobStatus := make([]int, len(stage.Jobs)) + + for i, j := range stage.Jobs { + statusCode, ok := statusMap[j.Status] + if !ok { + statusCode = -1 + } + jobStatus[i] = statusCode + } + var stageStatusCode int + for i, code := range jobStatus { + if i == 0 || code > stageStatusCode { + stageStatusCode = code + } + } + + for taskstatus, code := range statusMap { + if stageStatusCode == code { + stageStatus = taskstatus + break + } + } + stage.Status = stageStatus +} + +func (c *approveMap) setApproval(key string, value *approveWithLock) { + c.Lock() + defer c.Unlock() + c.m[key] = value +} + +func (c *approveMap) getApproval(key string) (*approveWithLock, bool) { + c.RLock() + defer c.RUnlock() + v, existed := c.m[key] + return v, existed +} +func (c *approveMap) deleteApproval(key string) { + c.Lock() + defer c.Unlock() + delete(c.m, key) +} + +func (c *approveWithLock) isApproval() (bool, int, error) { + c.Lock() + defer c.Unlock() + approveCount := 0 + for _, user := range c.approval.ApproveUsers { + if user.RejectOrApprove == config.Reject { + c.approval.RejectOrApprove = config.Reject + return false, approveCount, fmt.Errorf("%s reject this task", user.UserName) + } + if user.RejectOrApprove == config.Approve { + approveCount++ + } + } + if approveCount >= c.approval.NeededApprovers { + c.approval.RejectOrApprove = config.Approve + return true, approveCount, nil + } + return false, approveCount, nil +} + +func (c *approveWithLock) doApproval(userName, userID, comment string, appvove bool) error { + c.Lock() + defer c.Unlock() + for _, user := range c.approval.ApproveUsers { + if user.UserID != userID { + continue + } + if user.RejectOrApprove != "" { + return fmt.Errorf("%s have %s already", userName, user.RejectOrApprove) + } + user.Comment = comment + user.OperationTime = time.Now().Unix() + if appvove { + user.RejectOrApprove = config.Approve + return nil + } else { + user.RejectOrApprove = config.Reject + return nil + } + } + return fmt.Errorf("user %s has no authority to approve", userName) +} diff --git a/pkg/tool/remotedialer/session_windows.go b/pkg/microservice/aslan/core/common/service/workflowcontroller/stage_custom.go similarity index 34% rename from pkg/tool/remotedialer/session_windows.go rename to pkg/microservice/aslan/core/common/service/workflowcontroller/stage_custom.go index d43885bf4a6dd615d5d31c6c1f1ab4b22c9f467b..a24cc298e747bcd8343d132b50e37350c162f8e2 100644 --- a/pkg/tool/remotedialer/session_windows.go +++ b/pkg/microservice/aslan/core/common/service/workflowcontroller/stage_custom.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The KodeRover Authors. +Copyright 2022 The KodeRover Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,60 +14,42 @@ See the License for the specific language governing permissions and limitations under the License. */ -package remotedialer +package workflowcontroller import ( "context" - "time" - "github.com/gorilla/websocket" - "github.com/sirupsen/logrus" -) - -func (s *Session) startPingsWhileWindows(rootCtx context.Context) { - ctx, cancel := context.WithCancel(rootCtx) - s.pingCancel = cancel - s.pingWait.Add(1) - - go func() { - defer s.pingWait.Done() + "go.uber.org/zap" - t := time.NewTicker(PingWriteInterval) - defer t.Stop() + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller" +) - for { - select { - case <-ctx.Done(): - return - case <-t.C: - s.conn.Lock() - if err := s.conn.conn.WriteControl(websocket.PingMessage, []byte(""), time.Now().Add(time.Second)); err != nil { - logrus.WithError(err).Error("Error writing ping") - } - logrus.Debug("Wrote ping") - s.conn.Unlock() - } - } - }() +type CustomStageCtl struct { + stage *commonmodels.StageTask + workflowCtx *commonmodels.WorkflowTaskCtx + logger *zap.SugaredLogger + ack func() } -func (s *Session) ServeWhileWindows(ctx context.Context) (int, error) { - if s.client { - s.startPingsWhileWindows(ctx) +func NewCustomStageCtl(stage *commonmodels.StageTask, workflowCtx *commonmodels.WorkflowTaskCtx, logger *zap.SugaredLogger, ack func()) *CustomStageCtl { + return &CustomStageCtl{ + stage: stage, + logger: logger, + workflowCtx: workflowCtx, + ack: ack, } +} - for { - msType, reader, err := s.conn.NextReader() - if err != nil { - return 400, err - } - - if msType != websocket.BinaryMessage { - return 400, errWrongMessageType +func (c *CustomStageCtl) Run(ctx context.Context, concurrency int) { + var workerConcurrency = 1 + if c.stage.Parallel { + if len(c.stage.Jobs) > int(concurrency) { + workerConcurrency = int(concurrency) + } else { + workerConcurrency = len(c.stage.Jobs) } - if err := s.serveMessage(reader); err != nil { - return 500, err - } } + jobcontroller.RunJobs(ctx, c.stage.Jobs, c.workflowCtx, workerConcurrency, c.logger, c.ack) } diff --git a/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step.go b/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step.go new file mode 100644 index 0000000000000000000000000000000000000000..621276b320f38d853657af9447e2553d51b4b37b --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step.go @@ -0,0 +1,133 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stepcontroller + +import ( + "context" + "fmt" + "strings" + + "go.uber.org/zap" + "gopkg.in/yaml.v3" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" +) + +type StepCtl interface { + // get required info from aslan before job run. + PreRun(ctx context.Context) error + // run specific step, now only use for deploy job (freestyle job merge step together) + Run(ctx context.Context) (config.Status, error) + + // collect required info after job run.(like test report) + AfterRun(ctx context.Context) error +} + +func PrepareSteps(ctx context.Context, workflowCtx *commonmodels.WorkflowTaskCtx, jobPath *string, steps []*commonmodels.StepTask, logger *zap.SugaredLogger) error { + // inject global context into step spec. + workflowCtx.GlobalContextEach(func(k, v string) bool { + for _, step := range steps { + yamlString, err := yaml.Marshal(step.Spec) + if err != nil { + logger.Errorf("marshal step spec error: %v", err) + return false + } + replacedString := strings.ReplaceAll(string(yamlString), fmt.Sprintf("$(%s)", k), v) + if err := yaml.Unmarshal([]byte(replacedString), &step.Spec); err != nil { + logger.Errorf("unmarshal step spec error: %v", err) + return false + } + } + return true + }) + + stepCtls := []StepCtl{} + for _, step := range steps { + stepCtl, err := instantiateStepCtl(step, workflowCtx, jobPath, logger) + if err != nil { + return err + } + stepCtls = append(stepCtls, stepCtl) + } + for _, stepCtl := range stepCtls { + if err := stepCtl.PreRun(ctx); err != nil { + return err + } + } + return nil +} + +func RunSteps(ctx context.Context, workflowCtx *commonmodels.WorkflowTaskCtx, jobPath *string, steps []*commonmodels.StepTask, logger *zap.SugaredLogger) (config.Status, error) { + stepCtls := []StepCtl{} + for _, step := range steps { + stepCtl, err := instantiateStepCtl(step, workflowCtx, jobPath, logger) + if err != nil { + return config.StatusFailed, err + } + stepCtls = append(stepCtls, stepCtl) + } + for _, stepCtl := range stepCtls { + status, err := stepCtl.Run(ctx) + if err != nil || status != config.StatusPassed { + return status, err + } + } + return config.StatusPassed, nil +} + +func SummarizeSteps(ctx context.Context, workflowCtx *commonmodels.WorkflowTaskCtx, jobPath *string, steps []*commonmodels.StepTask, logger *zap.SugaredLogger) error { + stepCtls := []StepCtl{} + for _, step := range steps { + stepCtl, err := instantiateStepCtl(step, workflowCtx, jobPath, logger) + if err != nil { + return err + } + stepCtls = append(stepCtls, stepCtl) + } + for _, stepCtl := range stepCtls { + if err := stepCtl.AfterRun(ctx); err != nil { + return nil + } + } + return nil +} + +func instantiateStepCtl(step *commonmodels.StepTask, workflowCtx *commonmodels.WorkflowTaskCtx, jobPath *string, logger *zap.SugaredLogger) (StepCtl, error) { + var stepCtl StepCtl + var err error + switch step.StepType { + case config.StepGit: + stepCtl, err = NewGitCtl(step, logger) + case config.StepShell: + stepCtl, err = NewShellCtl(step, logger) + case config.StepDockerBuild: + stepCtl, err = NewDockerBuildCtl(step, logger) + case config.StepTools: + stepCtl, err = NewToolInstallCtl(step, jobPath, logger) + case config.StepArchive: + stepCtl, err = NewArchiveCtl(step, logger) + case config.StepDeploy: + stepCtl, err = NewDeployCtl(step, workflowCtx, logger) + case config.StepHelmDeploy: + stepCtl, err = NewHelmDeployCtl(step, workflowCtx, logger) + default: + logger.Errorf("unknown step type: %s", step.StepType) + return stepCtl, fmt.Errorf("unknown step type: %s", step.StepType) + } + return stepCtl, err +} diff --git a/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step_archive.go b/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step_archive.go new file mode 100644 index 0000000000000000000000000000000000000000..15e7e59355e9314d90278694931e7b8ca0b5b8cb --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step_archive.go @@ -0,0 +1,97 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stepcontroller + +import ( + "context" + "fmt" + + "go.uber.org/zap" + "gopkg.in/yaml.v3" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/types/step" +) + +type archiveCtl struct { + step *commonmodels.StepTask + archiveSpec *step.StepArchiveSpec + log *zap.SugaredLogger +} + +func NewArchiveCtl(stepTask *commonmodels.StepTask, log *zap.SugaredLogger) (*archiveCtl, error) { + yamlString, err := yaml.Marshal(stepTask.Spec) + if err != nil { + return nil, fmt.Errorf("marshal tool install spec error: %v", err) + } + archiveSpec := &step.StepArchiveSpec{} + if err := yaml.Unmarshal(yamlString, &archiveSpec); err != nil { + return nil, fmt.Errorf("unmarshal git spec error: %v", err) + } + stepTask.Spec = archiveSpec + return &archiveCtl{archiveSpec: archiveSpec, log: log, step: stepTask}, nil +} + +func (s *archiveCtl) PreRun(ctx context.Context) error { + if s.archiveSpec.S3 != nil { + return nil + } + var modelS3 *commonmodels.S3Storage + var err error + if s.archiveSpec.ObjectStorageID == "" { + modelS3, err = commonrepo.NewS3StorageColl().FindDefault() + if err != nil { + return err + } + s.archiveSpec.S3 = modelS3toS3(modelS3) + } else { + modelS3, err = commonrepo.NewS3StorageColl().Find(s.archiveSpec.ObjectStorageID) + if err != nil { + return err + } + s.archiveSpec.S3 = modelS3toS3(modelS3) + s.archiveSpec.S3.Subfolder = "" + } + s.step.Spec = s.archiveSpec + return nil +} + +func (s *archiveCtl) Run(ctx context.Context) (config.Status, error) { + return config.StatusPassed, nil +} + +func (s *archiveCtl) AfterRun(ctx context.Context) error { + return nil +} + +func modelS3toS3(modelS3 *commonmodels.S3Storage) *step.S3 { + resp := &step.S3{ + Ak: modelS3.Ak, + Sk: modelS3.Sk, + Endpoint: modelS3.Endpoint, + Bucket: modelS3.Bucket, + Subfolder: modelS3.Subfolder, + Insecure: modelS3.Insecure, + Provider: modelS3.Provider, + } + if modelS3.Insecure { + resp.Protocol = "http" + } + return resp +} diff --git a/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step_deploy.go b/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step_deploy.go new file mode 100644 index 0000000000000000000000000000000000000000..42ae355d84ca961f3ab955d5fbcdfd882b9a3325 --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step_deploy.go @@ -0,0 +1,364 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stepcontroller + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/pkg/errors" + "go.uber.org/zap" + "gopkg.in/yaml.v3" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" + "github.com/koderover/zadig/pkg/shared/kube/wrapper" + krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client" + "github.com/koderover/zadig/pkg/tool/kube/getter" + "github.com/koderover/zadig/pkg/tool/kube/updater" + "github.com/koderover/zadig/pkg/types/step" +) + +type deployCtl struct { + step *commonmodels.StepTask + deploySpec *step.StepDeploySpec + workflowCtx *commonmodels.WorkflowTaskCtx + namespace string + log *zap.SugaredLogger + kubeClient client.Client + restConfig *rest.Config +} + +func NewDeployCtl(stepTask *commonmodels.StepTask, workflowCtx *commonmodels.WorkflowTaskCtx, log *zap.SugaredLogger) (*deployCtl, error) { + yamlString, err := yaml.Marshal(stepTask.Spec) + if err != nil { + return nil, fmt.Errorf("marshal deploy spec error: %v", err) + } + deploySpec := &step.StepDeploySpec{} + if err := yaml.Unmarshal(yamlString, &deploySpec); err != nil { + return nil, fmt.Errorf("unmarshal deploy spec error: %v", err) + } + stepTask.Spec = deploySpec + return &deployCtl{deploySpec: deploySpec, workflowCtx: workflowCtx, log: log, step: stepTask}, nil +} + +func (s *deployCtl) PreRun(ctx context.Context) error { + return nil +} + +func (s *deployCtl) Run(ctx context.Context) (config.Status, error) { + if err := s.run(ctx); err != nil { + return config.StatusFailed, err + } + return s.wait(ctx) +} + +func (s *deployCtl) AfterRun(ctx context.Context) error { + return nil +} + +func (s *deployCtl) run(ctx context.Context) error { + var ( + err error + replaced = false + ) + env, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: s.workflowCtx.ProjectName, + EnvName: s.deploySpec.Env, + }) + if err != nil { + return err + } + s.namespace = env.Namespace + s.deploySpec.ClusterID = env.ClusterID + + if s.deploySpec.ClusterID != "" { + s.restConfig, err = kubeclient.GetRESTConfig(config.HubServerAddress(), s.deploySpec.ClusterID) + if err != nil { + err = errors.WithMessage(err, "can't get k8s rest config") + return err + } + + s.kubeClient, err = kubeclient.GetKubeClient(config.HubServerAddress(), s.deploySpec.ClusterID) + if err != nil { + err = errors.WithMessage(err, "can't init k8s client") + return err + } + } else { + s.kubeClient = krkubeclient.Client() + s.restConfig = krkubeclient.RESTConfig() + } + + // get servcie info + var ( + serviceInfo *commonmodels.Service + selector labels.Selector + ) + serviceInfo, err = commonrepo.NewServiceColl().Find( + &commonrepo.ServiceFindOption{ + ServiceName: s.deploySpec.ServiceName, + ProductName: s.workflowCtx.ProjectName, + ExcludeStatus: setting.ProductStatusDeleting, + Type: s.deploySpec.ServiceType, + }) + if err != nil { + // Maybe it is a share service, the entity is not under the project + serviceInfo, err = commonrepo.NewServiceColl().Find( + &commonrepo.ServiceFindOption{ + ServiceName: s.deploySpec.ServiceName, + ExcludeStatus: setting.ProductStatusDeleting, + Type: s.deploySpec.ServiceType, + }) + if err != nil { + return err + } + } + + if serviceInfo.WorkloadType == "" { + selector = labels.Set{setting.ProductLabel: s.workflowCtx.ProjectName, setting.ServiceLabel: s.deploySpec.ServiceName}.AsSelector() + + var deployments []*appsv1.Deployment + deployments, err = getter.ListDeployments(env.Namespace, selector, s.kubeClient) + if err != nil { + return err + } + + var statefulSets []*appsv1.StatefulSet + statefulSets, err = getter.ListStatefulSets(env.Namespace, selector, s.kubeClient) + if err != nil { + return err + } + + L: + for _, deploy := range deployments { + for _, container := range deploy.Spec.Template.Spec.Containers { + if container.Name == s.deploySpec.ServiceModule { + err = updater.UpdateDeploymentImage(deploy.Namespace, deploy.Name, s.deploySpec.ServiceModule, s.deploySpec.Image, s.kubeClient) + if err != nil { + err = errors.WithMessagef( + err, + "failed to update container image in %s/deployments/%s/%s", + env.Namespace, deploy.Name, container.Name) + return err + } + s.deploySpec.ReplaceResources = append(s.deploySpec.ReplaceResources, step.Resource{ + Kind: setting.Deployment, + Container: container.Name, + Origin: container.Image, + Name: deploy.Name, + }) + replaced = true + break L + } + } + } + Loop: + for _, sts := range statefulSets { + for _, container := range sts.Spec.Template.Spec.Containers { + if container.Name == s.deploySpec.ServiceModule { + err = updater.UpdateStatefulSetImage(sts.Namespace, sts.Name, s.deploySpec.ServiceModule, s.deploySpec.Image, s.kubeClient) + if err != nil { + err = errors.WithMessagef( + err, + "failed to update container image in %s/statefulsets/%s/%s", + env.Namespace, sts.Name, container.Name) + return err + } + s.deploySpec.ReplaceResources = append(s.deploySpec.ReplaceResources, step.Resource{ + Kind: setting.StatefulSet, + Container: container.Name, + Origin: container.Image, + Name: sts.Name, + }) + replaced = true + break Loop + } + } + } + } else { + switch serviceInfo.WorkloadType { + case setting.StatefulSet: + var statefulSet *appsv1.StatefulSet + statefulSet, _, err = getter.GetStatefulSet(env.Namespace, s.deploySpec.ServiceName, s.kubeClient) + if err != nil { + return err + } + for _, container := range statefulSet.Spec.Template.Spec.Containers { + if container.Name == s.deploySpec.ServiceModule { + err = updater.UpdateStatefulSetImage(statefulSet.Namespace, statefulSet.Name, s.deploySpec.ServiceModule, s.deploySpec.Image, s.kubeClient) + if err != nil { + err = errors.WithMessagef( + err, + "failed to update container image in %s/statefulsets/%s/%s", + env.Namespace, statefulSet.Name, container.Name) + return err + } + s.deploySpec.ReplaceResources = append(s.deploySpec.ReplaceResources, step.Resource{ + Kind: setting.StatefulSet, + Container: container.Name, + Origin: container.Image, + Name: statefulSet.Name, + }) + replaced = true + break + } + } + case setting.Deployment: + var deployment *appsv1.Deployment + deployment, _, err = getter.GetDeployment(env.Namespace, s.deploySpec.ServiceName, s.kubeClient) + if err != nil { + return err + } + for _, container := range deployment.Spec.Template.Spec.Containers { + if container.Name == s.deploySpec.ServiceModule { + err = updater.UpdateDeploymentImage(deployment.Namespace, deployment.Name, s.deploySpec.ServiceModule, s.deploySpec.Image, s.kubeClient) + if err != nil { + err = errors.WithMessagef( + err, + "failed to update container image in %s/deployments/%s/%s", + env.Namespace, deployment.Name, container.Name) + return err + } + s.deploySpec.ReplaceResources = append(s.deploySpec.ReplaceResources, step.Resource{ + Kind: setting.Deployment, + Container: container.Name, + Origin: container.Image, + Name: deployment.Name, + }) + replaced = true + break + } + } + } + } + if !replaced { + err = errors.Errorf("service %s container name %s is not found in env %s", s.deploySpec.ServiceName, s.deploySpec.ServiceModule, s.deploySpec.Env) + return err + } + s.step.Spec = s.deploySpec + return nil +} + +func (s *deployCtl) wait(ctx context.Context) (config.Status, error) { + timeout := time.After(time.Duration(s.timeout()) * time.Second) + + selector := labels.Set{setting.ProductLabel: s.workflowCtx.ProjectName, setting.ServiceLabel: s.deploySpec.ServiceName}.AsSelector() + + for { + select { + case <-ctx.Done(): + return config.StatusCancelled, nil + + case <-timeout: + pods, err := getter.ListPods(s.namespace, selector, s.kubeClient) + if err != nil { + return config.StatusFailed, err + } + + var msg []string + for _, pod := range pods { + podResource := wrapper.Pod(pod).Resource() + if podResource.Status != setting.StatusRunning && podResource.Status != setting.StatusSucceeded { + for _, cs := range podResource.ContainerStatuses { + // message为空不认为是错误状态，有可能还在waiting + if cs.Message != "" { + msg = append(msg, fmt.Sprintf("Status: %s, Reason: %s, Message: %s", cs.Status, cs.Reason, cs.Message)) + } + } + } + } + + if len(msg) != 0 { + err = errors.New(strings.Join(msg, "\n")) + } + + return config.StatusTimeout, err + + default: + time.Sleep(time.Second * 2) + ready := true + var err error + L: + for _, resource := range s.deploySpec.ReplaceResources { + switch resource.Kind { + case setting.Deployment: + d, found, e := getter.GetDeployment(s.namespace, resource.Name, s.kubeClient) + if e != nil { + err = e + } + if e != nil || !found { + s.log.Errorf( + "failed to check deployment ready status %s/%s/%s - %v", + s.namespace, + resource.Kind, + resource.Name, + e, + ) + ready = false + } else { + ready = wrapper.Deployment(d).Ready() + } + + if !ready { + break L + } + case setting.StatefulSet: + st, found, e := getter.GetStatefulSet(s.namespace, resource.Name, s.kubeClient) + if e != nil { + err = e + } + if err != nil || !found { + s.log.Errorf( + "failed to check statefulSet ready status %s/%s/%s", + s.namespace, + resource.Kind, + resource.Name, + e, + ) + ready = false + } else { + ready = wrapper.StatefulSet(st).Ready() + } + + if !ready { + break L + } + } + } + + if ready { + return config.StatusPassed, nil + } + } + } +} + +func (s *deployCtl) timeout() int { + if s.deploySpec.Timeout == 0 { + s.deploySpec.Timeout = setting.DeployTimeout + } + return s.deploySpec.Timeout +} diff --git a/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step_docker_build.go b/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step_docker_build.go new file mode 100644 index 0000000000000000000000000000000000000000..ca982f0934fa016100e79ff835edb44b5e1cf384 --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step_docker_build.go @@ -0,0 +1,91 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stepcontroller + +import ( + "context" + "fmt" + + "go.uber.org/zap" + "gopkg.in/yaml.v3" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/types/step" +) + +type dockerBuildCtl struct { + step *commonmodels.StepTask + dockerBuildSpec *step.StepDockerBuildSpec + log *zap.SugaredLogger +} + +func NewDockerBuildCtl(stepTask *commonmodels.StepTask, log *zap.SugaredLogger) (*dockerBuildCtl, error) { + yamlString, err := yaml.Marshal(stepTask.Spec) + if err != nil { + return nil, fmt.Errorf("marshal docker build spec error: %v", err) + } + dockerBuildSpec := &step.StepDockerBuildSpec{} + if err := yaml.Unmarshal(yamlString, &dockerBuildSpec); err != nil { + return nil, fmt.Errorf("unmarshal docker build spec error: %v", err) + } + if dockerBuildSpec.Proxy == nil { + dockerBuildSpec.Proxy = &step.Proxy{} + } + stepTask.Spec = dockerBuildSpec + return &dockerBuildCtl{dockerBuildSpec: dockerBuildSpec, log: log, step: stepTask}, nil +} + +func (s *dockerBuildCtl) PreRun(ctx context.Context) error { + if s.dockerBuildSpec.DockerRegistry != nil && s.dockerBuildSpec.DockerRegistry.DockerRegistryID != "" { + reg, _ := mongodb.NewRegistryNamespaceColl().Find(&mongodb.FindRegOps{ID: s.dockerBuildSpec.DockerRegistry.DockerRegistryID}) + s.dockerBuildSpec.DockerRegistry.UserName = reg.AccessKey + s.dockerBuildSpec.DockerRegistry.Password = reg.SecretKey + s.dockerBuildSpec.DockerRegistry.Namespace = reg.Namespace + s.dockerBuildSpec.DockerRegistry.Host = reg.RegAddr + } else { + s.dockerBuildSpec.DockerRegistry = &step.DockerRegistry{ + UserName: config.RegistryAccessKey(), + Password: config.RegistrySecretKey(), + Namespace: config.RegistryNamespace(), + Host: config.RegistryAddress(), + } + } + + proxies, _ := mongodb.NewProxyColl().List(&mongodb.ProxyArgs{}) + if len(proxies) != 0 { + s.dockerBuildSpec.Proxy.Address = proxies[0].Address + s.dockerBuildSpec.Proxy.EnableApplicationProxy = proxies[0].EnableApplicationProxy + s.dockerBuildSpec.Proxy.EnableRepoProxy = proxies[0].EnableRepoProxy + s.dockerBuildSpec.Proxy.NeedPassword = proxies[0].NeedPassword + s.dockerBuildSpec.Proxy.Password = proxies[0].Password + s.dockerBuildSpec.Proxy.Port = proxies[0].Port + s.dockerBuildSpec.Proxy.Type = proxies[0].Type + s.dockerBuildSpec.Proxy.Username = proxies[0].Username + } + s.step.Spec = s.dockerBuildSpec + return nil +} + +func (s *dockerBuildCtl) Run(ctx context.Context) (config.Status, error) { + return config.StatusPassed, nil +} + +func (s *dockerBuildCtl) AfterRun(ctx context.Context) error { + return nil +} diff --git a/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step_git.go b/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step_git.go new file mode 100644 index 0000000000000000000000000000000000000000..99998d6b39c4f66ba92dfbe746fc31ec0973fe70 --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step_git.go @@ -0,0 +1,96 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stepcontroller + +import ( + "context" + "fmt" + + "go.uber.org/zap" + "gopkg.in/yaml.v3" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types/step" +) + +type gitCtl struct { + step *commonmodels.StepTask + gitSpec *step.StepGitSpec + log *zap.SugaredLogger +} + +func NewGitCtl(stepTask *commonmodels.StepTask, log *zap.SugaredLogger) (*gitCtl, error) { + yamlString, err := yaml.Marshal(stepTask.Spec) + if err != nil { + return nil, fmt.Errorf("marshal git spec error: %v", err) + } + gitSpec := &step.StepGitSpec{} + if err := yaml.Unmarshal(yamlString, &gitSpec); err != nil { + return nil, fmt.Errorf("unmarshal git spec error: %v", err) + } + if gitSpec.Proxy == nil { + gitSpec.Proxy = &step.Proxy{} + } + stepTask.Spec = gitSpec + return &gitCtl{gitSpec: gitSpec, log: log, step: stepTask}, nil +} + +func (s *gitCtl) PreRun(ctx context.Context) error { + for _, repo := range s.gitSpec.Repos { + cID := repo.CodehostID + if cID == 0 { + log.Error("codehostID can't be empty") + return fmt.Errorf("codehostID can't be empty") + } + detail, err := systemconfig.New().GetCodeHost(cID) + if err != nil { + s.log.Error(err) + return err + } + repo.Source = detail.Type + repo.OauthToken = detail.AccessToken + repo.Address = detail.Address + repo.Username = detail.Username + repo.Password = detail.Password + repo.EnableProxy = detail.EnableProxy + } + proxies, _ := mongodb.NewProxyColl().List(&mongodb.ProxyArgs{}) + if len(proxies) != 0 { + s.gitSpec.Proxy.Address = proxies[0].Address + s.gitSpec.Proxy.EnableApplicationProxy = proxies[0].EnableApplicationProxy + s.gitSpec.Proxy.EnableRepoProxy = proxies[0].EnableRepoProxy + s.gitSpec.Proxy.NeedPassword = proxies[0].NeedPassword + s.gitSpec.Proxy.Password = proxies[0].Password + s.gitSpec.Proxy.Port = proxies[0].Port + s.gitSpec.Proxy.Type = proxies[0].Type + s.gitSpec.Proxy.Username = proxies[0].Username + } + s.step.Spec = s.gitSpec + return nil +} + +func (s *gitCtl) Run(ctx context.Context) (config.Status, error) { + return config.StatusPassed, nil +} + +func (s *gitCtl) AfterRun(ctx context.Context) error { + return nil +} diff --git a/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step_helm_deploy.go b/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step_helm_deploy.go new file mode 100644 index 0000000000000000000000000000000000000000..c2facba7fa2236ffeb658a8de469a5437e7ea664 --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step_helm_deploy.go @@ -0,0 +1,551 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stepcontroller + +import ( + "context" + "fmt" + "path/filepath" + "regexp" + "strings" + "time" + + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" + helmtool "github.com/koderover/zadig/pkg/tool/helmclient" + krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client" + s3tool "github.com/koderover/zadig/pkg/tool/s3" + "github.com/koderover/zadig/pkg/types/step" + "github.com/koderover/zadig/pkg/util/converter" + fsutil "github.com/koderover/zadig/pkg/util/fs" + yamlutil "github.com/koderover/zadig/pkg/util/yaml" + helmclient "github.com/mittwald/go-helm-client" + "github.com/pkg/errors" + "go.uber.org/zap" + "gopkg.in/yaml.v3" + "helm.sh/helm/v3/pkg/releaseutil" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + templatemodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3" + "github.com/koderover/zadig/pkg/setting" +) + +const ( + imageUrlParseRegexString = `(?P.+/)?(?P[^:]+){1}(:)?(?P.+)?` +) + +var ( + imageParseRegex = regexp.MustCompile(imageUrlParseRegexString) +) + +type helmDeployCtl struct { + step *commonmodels.StepTask + helmDeploySpec *step.StepHelmDeploySpec + workflowCtx *commonmodels.WorkflowTaskCtx + namespace string + log *zap.SugaredLogger + kubeClient client.Client + restConfig *rest.Config +} + +func NewHelmDeployCtl(stepTask *commonmodels.StepTask, workflowCtx *commonmodels.WorkflowTaskCtx, log *zap.SugaredLogger) (*helmDeployCtl, error) { + yamlString, err := yaml.Marshal(stepTask.Spec) + if err != nil { + return nil, fmt.Errorf("marshal helm deploy spec error: %v", err) + } + helmDeploySpec := &step.StepHelmDeploySpec{} + if err := yaml.Unmarshal(yamlString, &helmDeploySpec); err != nil { + return nil, fmt.Errorf("unmarshal helm deploy spec error: %v", err) + } + stepTask.Spec = helmDeploySpec + return &helmDeployCtl{helmDeploySpec: helmDeploySpec, workflowCtx: workflowCtx, log: log, step: stepTask}, nil +} + +func (s *helmDeployCtl) PreRun(ctx context.Context) error { + return nil +} + +func (s *helmDeployCtl) Run(ctx context.Context) (config.Status, error) { + if err := s.run(ctx); err != nil { + return config.StatusFailed, err + } + return config.StatusPassed, nil +} + +func (s *helmDeployCtl) AfterRun(ctx context.Context) error { + return nil +} + +func (s *helmDeployCtl) run(ctx context.Context) error { + var ( + err error + ) + + env, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: s.workflowCtx.ProjectName, + EnvName: s.helmDeploySpec.Env, + }) + if err != nil { + return err + } + s.namespace = env.Namespace + s.helmDeploySpec.ClusterID = env.ClusterID + + if s.helmDeploySpec.ClusterID != "" { + s.restConfig, err = kubeclient.GetRESTConfig(config.HubServerAddress(), s.helmDeploySpec.ClusterID) + if err != nil { + err = errors.WithMessage(err, "can't get k8s rest config") + return err + } + + s.kubeClient, err = kubeclient.GetKubeClient(config.HubServerAddress(), s.helmDeploySpec.ClusterID) + if err != nil { + err = errors.WithMessage(err, "can't init k8s client") + return err + } + } else { + s.kubeClient = krkubeclient.Client() + s.restConfig = krkubeclient.RESTConfig() + } + + // all involved containers + containerNameSet := sets.NewString() + for _, svcAndContainer := range s.helmDeploySpec.ImageAndModules { + singleContainerName := strings.TrimSuffix(svcAndContainer.ServiceModule, "_"+s.helmDeploySpec.ServiceName) + containerNameSet.Insert(singleContainerName) + } + + var ( + productInfo *commonmodels.Product + renderChart *templatemodels.RenderChart + replacedValuesYaml string + mergedValuesYaml string + replacedMergedValuesYaml string + servicePath string + chartPath string + replaceValuesMap map[string]interface{} + renderInfo *commonmodels.RenderSet + helmClient helmclient.Client + ) + + s.log.Infof("start helm deploy, productName %s serviceName %s containerName %v namespace %s", s.workflowCtx.ProjectName, + s.helmDeploySpec.ServiceName, containerNameSet.List(), s.namespace) + + productInfo, err = commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{Name: s.workflowCtx.ProjectName, EnvName: s.helmDeploySpec.Env}) + if err != nil { + err = errors.WithMessagef( + err, + "failed to get product %s/%s", + s.namespace, s.helmDeploySpec.ServiceName) + return err + } + + renderInfo, err = commonrepo.NewRenderSetColl().Find(&commonrepo.RenderSetFindOption{Name: productInfo.Render.Name, Revision: productInfo.Render.Revision}) + if err != nil { + err = errors.WithMessagef( + err, + "failed to get getRenderSet %s/%d", + productInfo.Render.Name, productInfo.Render.Revision) + return err + } + + serviceRevisionInProduct := int64(0) + involvedImagePaths := make(map[string]*commonmodels.ImagePathSpec) + for _, service := range productInfo.GetServiceMap() { + if service.ServiceName != s.helmDeploySpec.ServiceName { + continue + } + serviceRevisionInProduct = service.Revision + for _, container := range service.Containers { + if !containerNameSet.Has(container.Name) { + continue + } + if container.ImagePath == nil { + err = errors.WithMessagef(err, "image path of %s/%s is nil", service.ServiceName, container.Name) + return err + } + involvedImagePaths[container.Name] = container.ImagePath + } + break + } + + if len(involvedImagePaths) == 0 { + err = errors.Errorf("failed to find containers from service %s", s.helmDeploySpec.ServiceName) + return err + } + + for _, chartInfo := range renderInfo.ChartInfos { + if chartInfo.ServiceName == s.helmDeploySpec.ServiceName { + renderChart = chartInfo + break + } + } + + if renderChart == nil { + err = errors.Errorf("failed to update container image in %s/%s,chart not found", + s.namespace, s.helmDeploySpec.ServiceName) + return err + } + + defaultS3, err := s3.FindDefaultS3() + if err != nil { + return err + } + + defaultURL, err := defaultS3.GetEncryptedURL() + if err != nil { + return err + } + + // use revision of service currently applied in environment instead of the latest revision + path, errDownload := s.downloadService(s.workflowCtx.ProjectName, s.helmDeploySpec.ServiceName, defaultURL, serviceRevisionInProduct) + if errDownload != nil { + s.log.Warnf("failed to get chart of revision: %d for service: %s, use latest version", + serviceRevisionInProduct, s.helmDeploySpec.ServiceName) + path, errDownload = s.downloadService(s.workflowCtx.ProjectName, s.helmDeploySpec.ServiceName, + defaultURL, 0) + if errDownload != nil { + err = errors.WithMessagef( + errDownload, + "failed to download service %s/%s", + s.namespace, s.helmDeploySpec.ServiceName) + return err + } + } + + chartPath, err = fsutil.RelativeToCurrentPath(path) + if err != nil { + err = errors.WithMessagef( + err, + "failed to get relative path %s", + servicePath, + ) + return err + } + + serviceValuesYaml := renderChart.ValuesYaml + replaceValuesMap = make(map[string]interface{}) + + for _, svcAndContainer := range s.helmDeploySpec.ImageAndModules { + containerName := strings.TrimSuffix(svcAndContainer.ServiceModule, "_"+s.helmDeploySpec.ServiceName) + if imagePath, ok := involvedImagePaths[containerName]; ok { + validMatchData := getValidMatchData(imagePath) + singleReplaceValuesMap, errAssign := assignImageData(svcAndContainer.Image, validMatchData) + if errAssign != nil { + err = errors.WithMessagef( + errAssign, + "failed to pase image uri %s/%s", + s.namespace, s.helmDeploySpec.ServiceName) + return err + } + for k, v := range singleReplaceValuesMap { + replaceValuesMap[k] = v + } + } + } + + // replace image into service's values.yaml + replacedValuesYaml, err = replaceImage(serviceValuesYaml, replaceValuesMap) + if err != nil { + err = errors.WithMessagef( + err, + "failed to replace image uri %s/%s", + s.namespace, s.helmDeploySpec.ServiceName) + return err + } + if replacedValuesYaml == "" { + err = errors.Errorf("failed to set new image uri into service's values.yaml %s/%s", + s.namespace, s.helmDeploySpec.ServiceName) + return err + } + + // merge override values and kvs into service's yaml + mergedValuesYaml, err = helmtool.MergeOverrideValues(serviceValuesYaml, renderInfo.DefaultValues, renderChart.GetOverrideYaml(), renderChart.OverrideValues) + if err != nil { + err = errors.WithMessagef( + err, + "failed to merge override values %s", + renderChart.OverrideValues, + ) + return err + } + + // replace image into final merged values.yaml + replacedMergedValuesYaml, err = replaceImage(mergedValuesYaml, replaceValuesMap) + if err != nil { + err = errors.WithMessagef( + err, + "failed to replace image uri into helm values %s/%s", + s.namespace, s.helmDeploySpec.ServiceName) + return nil + } + if replacedMergedValuesYaml == "" { + err = errors.Errorf("failed to set image uri into mreged values.yaml in %s/%s", + s.namespace, s.helmDeploySpec.ServiceName) + return err + } + + s.log.Infof("final replaced merged values: \n%s", replacedMergedValuesYaml) + + helmClient, err = helmtool.NewClientFromNamespace(s.helmDeploySpec.ClusterID, s.namespace) + if err != nil { + err = errors.WithMessagef( + err, + "failed to create helm client %s/%s", + s.namespace, s.helmDeploySpec.ServiceName) + return err + } + + releaseName := s.helmDeploySpec.ReleaseName + + ensureUpgrade := func() error { + hrs, errHistory := helmClient.ListReleaseHistory(releaseName, 10) + if errHistory != nil { + // list history should not block deploy operation, error will be logged instead of returned + s.log.Errorf("failed to list release history, release: %s, err: %s", releaseName, errHistory) + return nil + } + if len(hrs) == 0 { + return nil + } + releaseutil.Reverse(hrs, releaseutil.SortByRevision) + rel := hrs[0] + + if rel.Info.Status.IsPending() { + return fmt.Errorf("failed to upgrade release: %s with exceptional status: %s", releaseName, rel.Info.Status) + } + return nil + } + + err = ensureUpgrade() + if err != nil { + return err + } + + timeOut := s.timeout() + chartSpec := helmclient.ChartSpec{ + ReleaseName: releaseName, + ChartName: chartPath, + Namespace: s.namespace, + ReuseValues: true, + Version: renderChart.ChartVersion, + ValuesYaml: replacedMergedValuesYaml, + SkipCRDs: false, + UpgradeCRDs: true, + Timeout: time.Second * time.Duration(timeOut), + Wait: true, + Replace: true, + MaxHistory: 10, + } + + done := make(chan bool) + go func(chan bool) { + if _, err = helmClient.InstallOrUpgradeChart(ctx, &chartSpec); err != nil { + err = errors.WithMessagef( + err, + "failed to upgrade helm chart %s/%s", + s.namespace, s.helmDeploySpec.ServiceName) + done <- false + } else { + done <- true + } + }(done) + + select { + case <-done: + break + case <-time.After(chartSpec.Timeout + time.Minute): + err = fmt.Errorf("failed to upgrade relase: %s, timeout", chartSpec.ReleaseName) + } + if err != nil { + return nil + } + + //替换环境变量中的chartInfos + for _, chartInfo := range renderInfo.ChartInfos { + if chartInfo.ServiceName == s.helmDeploySpec.ServiceName { + chartInfo.ValuesYaml = replacedValuesYaml + break + } + } + + // TODO too dangerous to override entire renderset! + if err := commonrepo.NewRenderSetColl().Update(&commonmodels.RenderSet{ + Name: renderInfo.Name, + Revision: renderInfo.Revision, + DefaultValues: renderInfo.DefaultValues, + ChartInfos: renderInfo.ChartInfos, + }); err != nil { + err = errors.WithMessagef( + err, + "failed to update renderset info %s/%s, renderset %s", + s.namespace, s.helmDeploySpec.ServiceName, renderInfo.Name) + return err + } + return nil +} + +func (s *helmDeployCtl) timeout() int { + if s.helmDeploySpec.Timeout == 0 { + s.helmDeploySpec.Timeout = setting.DeployTimeout + } + return s.helmDeploySpec.Timeout +} + +// download chart info of specific version, use the latest version if fails +func (s *helmDeployCtl) downloadService(productName, serviceName, storageURI string, revision int64) (string, error) { + logger := s.log + + fileName := serviceName + if revision > 0 { + fileName = fmt.Sprintf("%s-%d", serviceName, revision) + } + tarball := fmt.Sprintf("%s.tar.gz", fileName) + localBase := config.LocalServicePath(productName, serviceName) + tarFilePath := filepath.Join(localBase, tarball) + + exists, err := fsutil.FileExists(tarFilePath) + if err != nil { + return "", err + } + if exists { + return tarFilePath, nil + } + + s3Storage, err := s3.NewS3StorageFromEncryptedURI(storageURI) + if err != nil { + return "", err + } + + s3Storage.Subfolder = filepath.Join(s3Storage.Subfolder, config.ObjectStorageServicePath(productName, serviceName)) + forcedPathStyle := true + if s3Storage.Provider == setting.ProviderSourceAli { + forcedPathStyle = false + } + s3Client, err := s3tool.NewClient(s3Storage.Endpoint, s3Storage.Ak, s3Storage.Sk, s3Storage.Insecure, forcedPathStyle) + if err != nil { + s.log.Errorf("failed to create s3 client, err: %s", err) + return "", err + } + if err = s3Client.Download(s3Storage.Bucket, s3Storage.GetObjectPath(tarball), tarFilePath); err != nil { + logger.Errorf("failed to download file from s3, err: %s", err) + return "", err + } + + exists, err = fsutil.FileExists(tarFilePath) + if err != nil { + return "", err + } + if !exists { + return "", fmt.Errorf("file %s on s3 not found", s3Storage.GetObjectPath(tarball)) + } + return tarFilePath, nil +} + +func getValidMatchData(spec *commonmodels.ImagePathSpec) map[string]string { + ret := make(map[string]string) + if spec.Repo != "" { + ret[setting.PathSearchComponentRepo] = spec.Repo + } + if spec.Image != "" { + ret[setting.PathSearchComponentImage] = spec.Image + } + if spec.Tag != "" { + ret[setting.PathSearchComponentTag] = spec.Tag + } + return ret +} + +// assignImageData assign image url data into match data +// matchData: image=>absolute-path repo=>absolute-path tag=>absolute-path +// return: absolute-image-path=>image-value absolute-repo-path=>repo-value absolute-tag-path=>tag-value +func assignImageData(imageUrl string, matchData map[string]string) (map[string]interface{}, error) { + ret := make(map[string]interface{}) + // total image url assigned into one single value + if len(matchData) == 1 { + for _, v := range matchData { + ret[v] = imageUrl + } + return ret, nil + } + + resolvedImageUrl := resolveImageUrl(imageUrl) + + // image url assigned into repo/image+tag + if len(matchData) == 3 { + ret[matchData[setting.PathSearchComponentRepo]] = strings.TrimSuffix(resolvedImageUrl[setting.PathSearchComponentRepo], "/") + ret[matchData[setting.PathSearchComponentImage]] = resolvedImageUrl[setting.PathSearchComponentImage] + ret[matchData[setting.PathSearchComponentTag]] = resolvedImageUrl[setting.PathSearchComponentTag] + return ret, nil + } + + if len(matchData) == 2 { + // image url assigned into repo/image + tag + if tagPath, ok := matchData[setting.PathSearchComponentTag]; ok { + ret[tagPath] = resolvedImageUrl[setting.PathSearchComponentTag] + for k, imagePath := range matchData { + if k == setting.PathSearchComponentTag { + continue + } + ret[imagePath] = fmt.Sprintf("%s%s", resolvedImageUrl[setting.PathSearchComponentRepo], resolvedImageUrl[setting.PathSearchComponentImage]) + break + } + return ret, nil + } + // image url assigned into repo + image(tag) + ret[matchData[setting.PathSearchComponentRepo]] = strings.TrimSuffix(resolvedImageUrl[setting.PathSearchComponentRepo], "/") + ret[matchData[setting.PathSearchComponentImage]] = fmt.Sprintf("%s:%s", resolvedImageUrl[setting.PathSearchComponentImage], resolvedImageUrl[setting.PathSearchComponentTag]) + return ret, nil + } + + return nil, errors.Errorf("match data illegal, expect length: 1-3, actual length: %d", len(matchData)) +} + +// parse image url to map: repo=>xxx/xx/xx image=>xx tag=>xxx +func resolveImageUrl(imageUrl string) map[string]string { + subMatchAll := imageParseRegex.FindStringSubmatch(imageUrl) + result := make(map[string]string) + exNames := imageParseRegex.SubexpNames() + for i, matchedStr := range subMatchAll { + if i != 0 && matchedStr != "" && matchedStr != ":" { + result[exNames[i]] = matchedStr + } + } + return result +} + +// replace image defines in yaml by new version +func replaceImage(sourceYaml string, imageValuesMap map[string]interface{}) (string, error) { + nestedMap, err := converter.Expand(imageValuesMap) + if err != nil { + return "", err + } + bs, err := yaml.Marshal(nestedMap) + if err != nil { + return "", err + } + mergedBs, err := yamlutil.Merge([][]byte{[]byte(sourceYaml), bs}) + if err != nil { + return "", err + } + return string(mergedBs), nil +} diff --git a/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step_shell.go b/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step_shell.go new file mode 100644 index 0000000000000000000000000000000000000000..cecac78b6bd91c964c2a2eedf62cc4e8f03f5d48 --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step_shell.go @@ -0,0 +1,66 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stepcontroller + +import ( + "context" + "fmt" + "strings" + + "go.uber.org/zap" + "gopkg.in/yaml.v3" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/types/step" +) + +type shellCtl struct { + step *commonmodels.StepTask + shellSpec *step.StepShellSpec + log *zap.SugaredLogger +} + +func NewShellCtl(stepTask *commonmodels.StepTask, log *zap.SugaredLogger) (*shellCtl, error) { + yamlString, err := yaml.Marshal(stepTask.Spec) + if err != nil { + return nil, fmt.Errorf("marshal shell spec error: %v", err) + } + shellSpec := &step.StepShellSpec{} + if err := yaml.Unmarshal(yamlString, &shellSpec); err != nil { + return nil, fmt.Errorf("unmarshal shell spec error: %v", err) + } + stepTask.Spec = shellSpec + return &shellCtl{shellSpec: shellSpec, log: log, step: stepTask}, nil +} + +func (s *shellCtl) PreRun(ctx context.Context) error { + if len(s.shellSpec.Scripts) > 0 { + return nil + } + s.shellSpec.Scripts = strings.Split(replaceWrapLine(s.shellSpec.Script), "\n") + s.step.Spec = s.shellSpec + return nil +} + +func (s *shellCtl) Run(ctx context.Context) (config.Status, error) { + return config.StatusPassed, nil +} + +func (s *shellCtl) AfterRun(ctx context.Context) error { + return nil +} diff --git a/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step_tool_install.go b/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step_tool_install.go new file mode 100644 index 0000000000000000000000000000000000000000..4bd949a912169257c1f183e56184666ec85331b9 --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step_tool_install.go @@ -0,0 +1,135 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stepcontroller + +import ( + "context" + "fmt" + "strings" + + "go.uber.org/zap" + "gopkg.in/yaml.v3" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/types/step" +) + +type toolInstallCtl struct { + step *commonmodels.StepTask + toolInstalldSpec *step.StepToolInstallSpec + jobPath *string + log *zap.SugaredLogger +} + +func NewToolInstallCtl(stepTask *commonmodels.StepTask, jobPath *string, log *zap.SugaredLogger) (*toolInstallCtl, error) { + yamlString, err := yaml.Marshal(stepTask.Spec) + if err != nil { + return nil, fmt.Errorf("marshal tool install spec error: %v", err) + } + toolInstallSpec := &step.StepToolInstallSpec{} + if err := yaml.Unmarshal(yamlString, &toolInstallSpec); err != nil { + return nil, fmt.Errorf("unmarshal tool spec error: %v", err) + } + stepTask.Spec = toolInstallSpec + return &toolInstallCtl{toolInstalldSpec: toolInstallSpec, jobPath: jobPath, log: log, step: stepTask}, nil +} + +func (s *toolInstallCtl) PreRun(ctx context.Context) error { + spec := &step.StepToolInstallSpec{ + S3Storage: &step.S3{ + Ak: config.S3StorageAK(), + Sk: config.S3StorageSK(), + Endpoint: config.S3StorageEndpoint(), + Bucket: config.S3StorageBucket(), + Subfolder: config.S3StoragePath(), + Protocol: config.S3StorageProtocol(), + }, + } + objectStorage, _ := mongodb.NewS3StorageColl().FindDefault() + if objectStorage != nil { + spec.S3Storage.Endpoint = objectStorage.Endpoint + spec.S3Storage.Sk = objectStorage.Sk + spec.S3Storage.Ak = objectStorage.Ak + spec.S3Storage.Subfolder = objectStorage.Subfolder + spec.S3Storage.Bucket = objectStorage.Bucket + spec.S3Storage.Insecure = objectStorage.Insecure + spec.S3Storage.Protocol = "https" + if objectStorage.Insecure { + spec.S3Storage.Protocol = "http" + } + } + + for _, tool := range s.toolInstalldSpec.Installs { + install, err := buildInstallCtx(tool.Name, tool.Version) + if err != nil { + s.log.Error(err) + } + spec.Installs = append(spec.Installs, &step.Tool{ + Name: tool.Name, + Version: tool.Version, + BinPath: install.BinPath, + Download: install.DownloadPath, + Envs: install.Envs, + Scripts: strings.Split(replaceWrapLine(install.Scripts), "\n"), + }) + if *s.jobPath != "" { + *s.jobPath = strings.Join([]string{*s.jobPath, install.BinPath}, ":") + } else { + *s.jobPath = install.BinPath + } + } + + s.step.Spec = spec + return nil +} + +func (s *toolInstallCtl) Run(ctx context.Context) (config.Status, error) { + return config.StatusPassed, nil +} + +func (s *toolInstallCtl) AfterRun(ctx context.Context) error { + return nil +} + +//根据用户的配置和BuildStep中步骤的依赖，从系统配置的InstallItems中获取配置项，构建Install Context +func buildInstallCtx(name, version string) (*commonmodels.Install, error) { + resp := &commonmodels.Install{} + if name == "" || version == "" { + return resp, nil + } + install, err := commonrepo.NewInstallColl().Find(name, version) + if err != nil { + return resp, fmt.Errorf("%s:%s not found", name, version) + } + if !install.Enabled { + return resp, fmt.Errorf("%s:%s disabled", name, version) + } + resp = install + return resp, nil +} + +func replaceWrapLine(script string) string { + return strings.Replace(strings.Replace( + script, + "\r\n", + "\n", + -1, + ), "\r", "\n", -1) +} diff --git a/pkg/microservice/aslan/core/common/service/workflowcontroller/workflow.go b/pkg/microservice/aslan/core/common/service/workflowcontroller/workflow.go new file mode 100644 index 0000000000000000000000000000000000000000..6cea6f25f41609e9c513bdddf350bd41795fe61c --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/workflowcontroller/workflow.go @@ -0,0 +1,233 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package workflowcontroller + +import ( + "context" + "fmt" + "sync" + "time" + + uuid "github.com/satori/go.uuid" + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" +) + +var cancelChannelMap sync.Map + +type workflowCtl struct { + workflowTask *commonmodels.WorkflowTask + globalContextMutex sync.RWMutex + logger *zap.SugaredLogger + ack func() +} + +func NewWorkflowController(workflowTask *commonmodels.WorkflowTask, logger *zap.SugaredLogger) *workflowCtl { + ctl := &workflowCtl{ + workflowTask: workflowTask, + logger: logger, + } + ctl.ack = ctl.updateWorkflowTask + return ctl +} + +func CancelWorkflowTask(userName, workflowName string, taskID int64, logger *zap.SugaredLogger) error { + t, err := commonrepo.NewworkflowTaskv4Coll().Find(workflowName, taskID) + if err != nil { + logger.Errorf("[%s] task: %s:%d not found", userName, workflowName, taskID) + return err + } + + if t.Status == config.StatusPassed { + logger.Errorf("[%s] task: %s:%d is passed, cannot cancel", userName, workflowName, taskID) + return fmt.Errorf("task: %s:%d is passed, cannot cancel", workflowName, taskID) + } + + t.Status = config.StatusCancelled + t.TaskRevoker = userName + + logger.Infof("[%s] CancelRunningTask %s:%d", userName, taskID, taskID) + + if err := commonrepo.NewworkflowTaskv4Coll().Update(t.ID.Hex(), t); err != nil { + logger.Errorf("[%s] update task: %s:%d error: %v", userName, workflowName, taskID, err) + return err + } + + q := ConvertTaskToQueue(t) + if err := Remove(q); err != nil { + logger.Errorf("[%s] remove queue task: %s:%d error: %v", userName, workflowName, taskID, err) + return err + } + + value, ok := cancelChannelMap.Load(fmt.Sprintf("%s-%d", workflowName, taskID)) + if !ok { + logger.Errorf("no mactched task found, id: %d, workflow name: %s", taskID, workflowName) + return nil + } + if f, ok := value.(context.CancelFunc); ok { + f() + return nil + } + return fmt.Errorf("cancel func type mismatched, id: %d, workflow name: %s", taskID, workflowName) +} + +func (c *workflowCtl) Run(ctx context.Context, concurrency int) { + if c.workflowTask.GlobalContext == nil { + c.workflowTask.GlobalContext = make(map[string]string) + } + c.workflowTask.Status = config.StatusRunning + c.workflowTask.StartTime = time.Now().Unix() + c.ack() + c.logger.Infof("start workflow: %s,status: %s", c.workflowTask.WorkflowName, c.workflowTask.Status) + defer func() { + c.workflowTask.EndTime = time.Now().Unix() + c.logger.Infof("finish workflow: %s,status: %s", c.workflowTask.WorkflowName, c.workflowTask.Status) + c.ack() + }() + ctx, cancel := context.WithCancel(ctx) + defer cancel() + cancelKey := fmt.Sprintf("%s-%d", c.workflowTask.WorkflowName, c.workflowTask.TaskID) + cancelChannelMap.Store(cancelKey, cancel) + defer cancelChannelMap.Delete(cancelKey) + + workflowCtx := &commonmodels.WorkflowTaskCtx{ + WorkflowName: c.workflowTask.WorkflowName, + ProjectName: c.workflowTask.ProjectName, + TaskID: c.workflowTask.TaskID, + Workspace: "/workspace", + DistDir: fmt.Sprintf("%s/%s/dist/%d", config.S3StoragePath(), c.workflowTask.WorkflowName, c.workflowTask.TaskID), + DockerMountDir: fmt.Sprintf("/tmp/%s/docker/%d", uuid.NewV4(), time.Now().Unix()), + ConfigMapMountDir: fmt.Sprintf("/tmp/%s/cm/%d", uuid.NewV4(), time.Now().Unix()), + WorkflowKeyVals: c.workflowTask.KeyVals, + GlobalContextGet: c.getGlobalContext, + GlobalContextSet: c.setGlobalContext, + GlobalContextEach: c.globalContextEach, + } + + RunStages(ctx, c.workflowTask.Stages, workflowCtx, concurrency, c.logger, c.ack) + updateworkflowStatus(c.workflowTask) +} + +func updateworkflowStatus(workflow *commonmodels.WorkflowTask) { + statusMap := map[config.Status]int{ + config.StatusReject: 5, + config.StatusCancelled: 4, + config.StatusTimeout: 3, + config.StatusFailed: 2, + config.StatusPassed: 1, + config.StatusSkipped: 0, + } + + // 初始化workflowStatus为创建状态 + workflowStatus := config.StatusRunning + + stageStatus := make([]int, len(workflow.Stages)) + + for i, j := range workflow.Stages { + statusCode, ok := statusMap[j.Status] + if !ok { + statusCode = -1 + } + stageStatus[i] = statusCode + } + var workflowStatusCode int + for i, code := range stageStatus { + if i == 0 || code > workflowStatusCode { + workflowStatusCode = code + } + } + + for taskstatus, code := range statusMap { + if workflowStatusCode == code { + workflowStatus = taskstatus + break + } + } + workflow.Status = workflowStatus +} + +func (c *workflowCtl) updateWorkflowTask() { + taskInColl, err := commonrepo.NewworkflowTaskv4Coll().Find(c.workflowTask.WorkflowName, c.workflowTask.TaskID) + if err != nil { + c.logger.Errorf("find workflow task v4 %s failed,error: %v", c.workflowTask.WorkflowName, err) + return + } + // 如果当前状态已经通过或者失败, 不处理新接受到的ACK + if taskInColl.Status == config.StatusPassed || taskInColl.Status == config.StatusFailed || taskInColl.Status == config.StatusTimeout { + c.logger.Infof("%s:%d:%s task already done", c.workflowTask.WorkflowName, c.workflowTask.TaskID, taskInColl.Status) + return + } + // 如果当前状态已经是取消状态, 一般为用户取消了任务, 此时任务在短暂时间内会继续运行一段时间, + if taskInColl.Status == config.StatusCancelled { + // Task终止状态可能为Pass, Fail, Cancel, Timeout + // backend 会继续接受到ACK, 在这种情况下, 终止状态之外的ACK都无需处理，避免出现取消之后又被重置成运行态 + if c.workflowTask.Status != config.StatusFailed && c.workflowTask.Status != config.StatusPassed && c.workflowTask.Status != config.StatusCancelled && c.workflowTask.Status != config.StatusTimeout { + c.logger.Infof("%s:%d task has been cancelled, ACK dropped", c.workflowTask.WorkflowName, c.workflowTask.TaskID) + return + } + } + if success := UpdateQueue(c.workflowTask); !success { + c.logger.Errorf("%s:%d update t status error", c.workflowTask.WorkflowName, c.workflowTask.TaskID) + } + // TODO update workflow task + if err := commonrepo.NewworkflowTaskv4Coll().Update(c.workflowTask.ID.Hex(), c.workflowTask); err != nil { + c.logger.Errorf("update workflow task v4 failed,error: %v", err) + } + + if c.workflowTask.Status == config.StatusPassed || c.workflowTask.Status == config.StatusFailed || c.workflowTask.Status == config.StatusTimeout || c.workflowTask.Status == config.StatusCancelled { + c.logger.Infof("%s:%d:%v task done", c.workflowTask.WorkflowName, c.workflowTask.TaskID, c.workflowTask.Status) + q := ConvertTaskToQueue(c.workflowTask) + if err := Remove(q); err != nil { + c.logger.Errorf("remove queue task: %s:%d error: %v", c.workflowTask.WorkflowName, c.workflowTask.TaskID, err) + } + result, err := commonrepo.NewStrategyColl().GetByTarget(commonmodels.WorkflowTaskRetention) + if err != nil { + c.logger.Errorf("get workflow task retention strategy error: %s", err) + result = commonmodels.DefaultWorkflowTaskRetention + } + if err = commonrepo.NewworkflowTaskv4Coll().ArchiveHistoryWorkflowTask(c.workflowTask.WorkflowName, result.Retention.MaxItems, result.Retention.MaxDays); err != nil { + c.logger.Errorf("ArchiveHistoryWorkflowTask error: %v", err) + } + } + +} + +func (c *workflowCtl) getGlobalContext(key string) (string, bool) { + c.globalContextMutex.RLock() + defer c.globalContextMutex.RUnlock() + v, existed := c.workflowTask.GlobalContext[key] + return v, existed +} + +func (c *workflowCtl) setGlobalContext(key, value string) { + c.globalContextMutex.Lock() + defer c.globalContextMutex.Unlock() + c.workflowTask.GlobalContext[key] = value +} + +func (c *workflowCtl) globalContextEach(f func(k, v string) bool) { + c.globalContextMutex.RLock() + defer c.globalContextMutex.RUnlock() + for k, v := range c.workflowTask.GlobalContext { + if !f(k, v) { + return + } + } +} diff --git a/pkg/microservice/aslan/core/common/util/validate.go b/pkg/microservice/aslan/core/common/util/validate.go new file mode 100644 index 0000000000000000000000000000000000000000..4a1f625828f1f3ea4db8adc1aad38ed52282b1c2 --- /dev/null +++ b/pkg/microservice/aslan/core/common/util/validate.go @@ -0,0 +1,36 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + + "github.com/koderover/zadig/pkg/setting" +) + +func CheckDefineResourceParam(req setting.Request, reqSpec setting.RequestSpec) error { + if req != setting.DefineRequest { + return nil + } + if reqSpec.CpuLimit < 1 { + return fmt.Errorf("Parameter res_req_spc.cpu_limit must be greater than 1m") + } + if reqSpec.MemoryLimit < 1 { + return fmt.Errorf("Parameter res_req_spc.memory_limit must be greater than 1Mi") + } + return nil +} diff --git a/pkg/microservice/aslan/core/cron/handler/router.go b/pkg/microservice/aslan/core/cron/handler/router.go index 9d06de008e7fe9c66cb1428ef4c9303fd2ab40f3..29bb3f483538eacb681f1ffafe1918a7365229d9 100644 --- a/pkg/microservice/aslan/core/cron/handler/router.go +++ b/pkg/microservice/aslan/core/cron/handler/router.go @@ -18,14 +18,11 @@ package handler import ( "github.com/gin-gonic/gin" - - gin2 "github.com/koderover/zadig/pkg/middleware/gin" ) type Router struct{} func (*Router) Inject(router *gin.RouterGroup) { - router.Use(gin2.Auth()) // --------------------------------------------------------------------------------------- // 定时任务管理接口 diff --git a/pkg/microservice/aslan/core/cron/service/cron.go b/pkg/microservice/aslan/core/cron/service/cron.go index be8d7487401a4fcf49cae087ce7806442f05613e..fc53f2aad1fdd9c57ef4a0cff226eadf92f87340 100644 --- a/pkg/microservice/aslan/core/cron/service/cron.go +++ b/pkg/microservice/aslan/core/cron/service/cron.go @@ -39,9 +39,11 @@ func CleanJobCronJob(log *zap.SugaredLogger) { workflowSelector := labels.Set{"p-type": "workflow"}.AsSelector() singleSelector := labels.Set{"p-type": "single"}.AsSelector() testSelector := labels.Set{"p-type": "test"}.AsSelector() + artifactSelector := labels.Set{"p-type": "artifact"}.AsSelector() cleanJob(config.Namespace(), workflowSelector, kubeClient, log) cleanJob(config.Namespace(), singleSelector, kubeClient, log) cleanJob(config.Namespace(), testSelector, kubeClient, log) + cleanJob(config.Namespace(), artifactSelector, kubeClient, log) cleanServiceJob(kubeClient, log) log.Infof("finnish clean job...") } @@ -52,9 +54,11 @@ func CleanConfigmapCronJob(log *zap.SugaredLogger) { workflowSelector := labels.Set{"p-type": "workflow"}.AsSelector() singleSelector := labels.Set{"p-type": "single"}.AsSelector() testSelector := labels.Set{"p-type": "test"}.AsSelector() + artifactSelector := labels.Set{"p-type": "artifact"}.AsSelector() cleanConfigmap(config.Namespace(), workflowSelector, kubeClient, log) cleanConfigmap(config.Namespace(), singleSelector, kubeClient, log) cleanConfigmap(config.Namespace(), testSelector, kubeClient, log) + cleanConfigmap(config.Namespace(), artifactSelector, kubeClient, log) cleanServiceConfigmap(kubeClient, log) log.Infof("finnish clean configmap...") } diff --git a/pkg/microservice/aslan/core/delivery/handler/artifact.go b/pkg/microservice/aslan/core/delivery/handler/artifact.go index a8eed0c3d75207244af2d38d7054337e5e5220b1..2e84ec6677d9e8212dc374b8332ed85adfbcdef9 100644 --- a/pkg/microservice/aslan/core/delivery/handler/artifact.go +++ b/pkg/microservice/aslan/core/delivery/handler/artifact.go @@ -77,6 +77,22 @@ func ListDeliveryArtifacts(c *gin.Context) { ctx.Resp, ctx.Err = artifacts, err } +func GetDeliveryArtifactIDByImage(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + image := c.Query("image") + if image == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("image can't be empty!") + return + } + args := &commonrepo.DeliveryArtifactArgs{ + Image: image, + } + + ctx.Resp, ctx.Err = deliveryservice.GetDeliveryArtifactIDByImage(args, ctx.Logger) +} + func GetDeliveryArtifact(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() @@ -89,7 +105,6 @@ func GetDeliveryArtifact(c *gin.Context) { args := &commonrepo.DeliveryArtifactArgs{ ID: id, } - args.ID = id ctx.Resp, ctx.Err = deliveryservice.GetDeliveryArtifact(args, ctx.Logger) } diff --git a/pkg/microservice/aslan/core/delivery/handler/product.go b/pkg/microservice/aslan/core/delivery/handler/product.go index 59eb70b2ce7a7e9ce50c8befd487d54514561f9a..43bf3bfda49ca583cefa3d9fc10989e096b6bc05 100644 --- a/pkg/microservice/aslan/core/delivery/handler/product.go +++ b/pkg/microservice/aslan/core/delivery/handler/product.go @@ -17,8 +17,6 @@ limitations under the License. package handler import ( - "strconv" - "github.com/gin-gonic/gin" deliveryservice "github.com/koderover/zadig/pkg/microservice/aslan/core/delivery/service" @@ -26,19 +24,6 @@ import ( e "github.com/koderover/zadig/pkg/tool/errors" ) -func ListDeliveryProduct(c *gin.Context) { - ctx := internalhandler.NewContext(c) - defer func() { internalhandler.JSONResponse(c, ctx) }() - //params validate - orgIDStr := c.Query("orgId") - orgID, err := strconv.Atoi(orgIDStr) - if err != nil { - ctx.Err = e.ErrInvalidParam.AddDesc("orgId can't be empty!") - return - } - ctx.Resp, ctx.Err = deliveryservice.FindDeliveryProduct(orgID, ctx.Logger) -} - func GetProductByDeliveryInfo(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() @@ -49,5 +34,5 @@ func GetProductByDeliveryInfo(c *gin.Context) { return } - ctx.Resp, ctx.Err = deliveryservice.GetProductByDeliveryInfo(ctx.Username, releaseID, ctx.Logger) + ctx.Resp, ctx.Err = deliveryservice.GetProductByDeliveryInfo(ctx.UserName, releaseID, ctx.Logger) } diff --git a/pkg/microservice/aslan/core/delivery/handler/router.go b/pkg/microservice/aslan/core/delivery/handler/router.go index cb6b92e50c8457bf0a804fa99ed0509fa2a0db1a..4c29b15f93ae963e9ab5bc3fbe92c2d0142af6b7 100644 --- a/pkg/microservice/aslan/core/delivery/handler/router.go +++ b/pkg/microservice/aslan/core/delivery/handler/router.go @@ -18,20 +18,17 @@ package handler import ( "github.com/gin-gonic/gin" - - gin2 "github.com/koderover/zadig/pkg/middleware/gin" - "github.com/koderover/zadig/pkg/types/permission" ) type Router struct{} func (*Router) Inject(router *gin.RouterGroup) { - router.Use(gin2.Auth()) deliveryArtifact := router.Group("artifacts") { deliveryArtifact.GET("", ListDeliveryArtifacts) deliveryArtifact.GET("/:id", GetDeliveryArtifact) + deliveryArtifact.GET("/image", GetDeliveryArtifactIDByImage) deliveryArtifact.POST("", CreateDeliveryArtifacts) deliveryArtifact.POST("/:id", UpdateDeliveryArtifact) deliveryArtifact.POST("/:id/activities", CreateDeliveryActivities) @@ -39,7 +36,6 @@ func (*Router) Inject(router *gin.RouterGroup) { deliveryProduct := router.Group("products") { - deliveryProduct.GET("", ListDeliveryProduct) deliveryProduct.GET("/:releaseId", GetProductByDeliveryInfo) } @@ -47,7 +43,14 @@ func (*Router) Inject(router *gin.RouterGroup) { { deliveryRelease.GET("/:id", GetDeliveryVersion) deliveryRelease.GET("", ListDeliveryVersion) - deliveryRelease.DELETE("/:id", GetProductNameByDelivery, gin2.IsHavePermission([]string{permission.ReleaseDeleteUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, DeleteDeliveryVersion) + deliveryRelease.DELETE("/:id", GetProductNameByDelivery, DeleteDeliveryVersion) + deliveryRelease.POST("/helm", CreateHelmDeliveryVersion) + deliveryRelease.POST("/helm/global-variables", ApplyDeliveryGlobalVariables) + deliveryRelease.GET("/helm/charts", DownloadDeliveryChart) + deliveryRelease.GET("/helm/charts/version", GetChartVersionFromRepo) + deliveryRelease.GET("/helm/charts/preview", PreviewGetDeliveryChart) + deliveryRelease.GET("/helm/charts/filePath", GetDeliveryChartFilePath) + deliveryRelease.GET("/helm/charts/fileContent", GetDeliveryChartFileContent) } deliveryPackage := router.Group("packages") diff --git a/pkg/microservice/aslan/core/delivery/handler/version.go b/pkg/microservice/aslan/core/delivery/handler/version.go index 28e26ff0edc79922bc970434c88dcf742baeb899..0039a6f1b43817a6a76d9c1934950ef72cb11034 100644 --- a/pkg/microservice/aslan/core/delivery/handler/version.go +++ b/pkg/microservice/aslan/core/delivery/handler/version.go @@ -19,17 +19,14 @@ package handler import ( "encoding/json" "fmt" - "strconv" + "net/http" "strings" "github.com/gin-gonic/gin" "github.com/koderover/zadig/pkg/microservice/aslan/config" - commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" deliveryservice "github.com/koderover/zadig/pkg/microservice/aslan/core/delivery/service" - workflowservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" @@ -65,196 +62,31 @@ func GetDeliveryVersion(c *gin.Context) { } version := new(commonrepo.DeliveryVersionArgs) version.ID = ID - ctx.Resp, ctx.Err = deliveryservice.GetDeliveryVersion(version, ctx.Logger) -} - -type ReleaseInfo struct { - VersionInfo *commonmodels.DeliveryVersion `json:"versionInfo"` - BuildInfo []*commonmodels.DeliveryBuild `json:"buildInfo"` - DeployInfo []*commonmodels.DeliveryDeploy `json:"deployInfo"` - TestInfo []*commonmodels.DeliveryTest `json:"testInfo"` - DistributeInfo []*commonmodels.DeliveryDistribute `json:"distributeInfo"` - SecurityInfo []*DeliverySecurityStats `json:"securityStatsInfo"` -} - -type DeliverySecurityStats struct { - ImageName string `json:"imageName"` - ImageID string `json:"imageId"` - DeliverySecurityStatsInfo DeliverySecurityStatsInfo `json:"deliverySecurityStatsInfo"` -} - -type DeliverySecurityStatsInfo struct { - Total int `json:"total"` - Unknown int `json:"unkown"` - Negligible int `json:"negligible"` - Low int `json:"low"` - Medium int `json:"medium"` - High int `json:"high"` - Critical int `json:"critical"` + ctx.Resp, ctx.Err = deliveryservice.GetDetailReleaseData(version, ctx.Logger) } func ListDeliveryVersion(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - //params validate - orgIDStr := c.Query("orgId") - orgID, err := strconv.Atoi(orgIDStr) + + args := new(deliveryservice.ListDeliveryVersionArgs) + err := c.BindQuery(args) if err != nil { - ctx.Err = e.ErrInvalidParam.AddDesc("orgId can't be empty!") + ctx.Err = e.ErrInvalidParam.AddErr(err) return } - taskIDStr := c.Query("taskId") - var taskID = 0 - if taskIDStr != "" { - taskID, err = strconv.Atoi(taskIDStr) - if err != nil { - ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) - return - } + if args.Page <= 0 { + args.Page = 1 } - - perPageStr := c.Query("per_page") - pageStr := c.Query("page") - var ( - perPage int - page int - ) - if perPageStr == "" { - perPage = 20 - } else { - perPage, err = strconv.Atoi(perPageStr) - if err != nil { - ctx.Err = e.ErrInvalidParam.AddDesc(fmt.Sprintf("perPage args err :%s", err)) - return - } + if args.PerPage <= 0 { + args.PerPage = 20 } - - if pageStr == "" { - page = 1 - } else { - page, err = strconv.Atoi(pageStr) - if err != nil { - ctx.Err = e.ErrInvalidParam.AddDesc(fmt.Sprintf("page args err :%s", err)) - return - } + if len(args.Verbosity) == 0 { + args.Verbosity = deliveryservice.VerbosityDetailed } - serviceName := c.Query("serviceName") - - version := new(commonrepo.DeliveryVersionArgs) - version.OrgID = orgID - version.ProductName = c.Query("productName") - version.WorkflowName = c.Query("workflowName") - version.TaskID = taskID - version.PerPage = perPage - version.Page = page - deliveryVersions, err := deliveryservice.FindDeliveryVersion(version, ctx.Logger) - - if err != nil { - ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) - return - } - releaseInfos := make([]*ReleaseInfo, 0) - for _, deliveryVersion := range deliveryVersions { - releaseInfo := new(ReleaseInfo) - //versionInfo - releaseInfo.VersionInfo = deliveryVersion - - //deployInfo - deliveryDeployArgs := new(commonrepo.DeliveryDeployArgs) - deliveryDeployArgs.ReleaseID = deliveryVersion.ID.Hex() - deliveryDeploys, err := deliveryservice.FindDeliveryDeploy(deliveryDeployArgs, ctx.Logger) - if err != nil { - ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) - } - // 查询条件serviceName - if serviceName != "" { - match := false - for _, deliveryDeploy := range deliveryDeploys { - if deliveryDeploy.ServiceName == serviceName { - match = true - break - } - } - if !match { - continue - } - } - // 将serviceName替换为服务名/服务组件的形式，用于前端展示 - for _, deliveryDeploy := range deliveryDeploys { - if deliveryDeploy.ContainerName != "" { - deliveryDeploy.ServiceName = deliveryDeploy.ServiceName + "/" + deliveryDeploy.ContainerName - } - } - releaseInfo.DeployInfo = deliveryDeploys - - //buildInfo - deliveryBuildArgs := new(commonrepo.DeliveryBuildArgs) - deliveryBuildArgs.ReleaseID = deliveryVersion.ID.Hex() - deliveryBuilds, err := deliveryservice.FindDeliveryBuild(deliveryBuildArgs, ctx.Logger) - if err != nil { - ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) - } - releaseInfo.BuildInfo = deliveryBuilds - - //testInfo - deliveryTestArgs := new(commonrepo.DeliveryTestArgs) - deliveryTestArgs.ReleaseID = deliveryVersion.ID.Hex() - deliveryTests, err := deliveryservice.FindDeliveryTest(deliveryTestArgs, ctx.Logger) - if err != nil { - ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) - } - releaseInfo.TestInfo = deliveryTests - - //securityStatsInfo - deliverySecurityStatss := make([]*DeliverySecurityStats, 0) - if pipelineTask, err := workflowservice.GetPipelineTaskV2(int64(deliveryVersion.TaskID), deliveryVersion.WorkflowName, config.WorkflowType, ctx.Logger); err == nil { - for _, subStage := range pipelineTask.Stages { - if subStage.TaskType == config.TaskSecurity { - subSecurityTaskMap := subStage.SubTasks - for _, subTask := range subSecurityTaskMap { - securityInfo, _ := base.ToSecurityTask(subTask) - - deliverySecurityStats := new(DeliverySecurityStats) - deliverySecurityStats.ImageName = securityInfo.ImageName - deliverySecurityStats.ImageID = securityInfo.ImageID - deliverySecurityStatsMap, err := deliveryservice.FindDeliverySecurityStatistics(securityInfo.ImageID, ctx.Logger) - if err != nil { - ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) - } - var transErr error - b, err := json.Marshal(deliverySecurityStatsMap) - if err != nil { - transErr = fmt.Errorf("marshal task error: %v", err) - } - - if err := json.Unmarshal(b, &deliverySecurityStats.DeliverySecurityStatsInfo); err != nil { - transErr = fmt.Errorf("unmarshal task error: %v", err) - } - if transErr != nil { - ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) - } - - deliverySecurityStatss = append(deliverySecurityStatss, deliverySecurityStats) - } - break - } - } - releaseInfo.SecurityInfo = deliverySecurityStatss - } - - //distributeInfo - deliveryDistributeArgs := new(commonrepo.DeliveryDistributeArgs) - deliveryDistributeArgs.ReleaseID = deliveryVersion.ID.Hex() - deliveryDistributes, _ := deliveryservice.FindDeliveryDistribute(deliveryDistributeArgs, ctx.Logger) - - releaseInfo.DistributeInfo = deliveryDistributes - - releaseInfos = append(releaseInfos, releaseInfo) - } - ctx.Err = err - ctx.Resp = releaseInfos + ctx.Resp, ctx.Err = deliveryservice.ListDeliveryVersion(args, ctx.Logger) } func getFileName(fileName string) string { @@ -286,16 +118,9 @@ type DeliveryFileInfo struct { func ListPackagesVersion(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - //params validate - orgIDStr := c.Query("orgId") - orgID, err := strconv.Atoi(orgIDStr) - if err != nil { - ctx.Err = e.ErrInvalidParam.AddDesc("orgId can't be empty!") - return - } + version := &commonrepo.DeliveryVersionArgs{ - OrgID: orgID, - ProductName: c.Query("productName"), + ProductName: c.Query("projectName"), } deliveryVersions, err := deliveryservice.FindDeliveryVersion(version, ctx.Logger) if err != nil { @@ -340,10 +165,28 @@ func ListPackagesVersion(c *gin.Context) { ctx.Resp = fileInfoList } +func CreateHelmDeliveryVersion(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(deliveryservice.CreateHelmDeliveryVersionArgs) + err := c.ShouldBindJSON(args) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + args.CreateBy = ctx.UserName + + bs, _ := json.Marshal(args) + internalhandler.InsertOperationLog(c, ctx.UserName, args.ProductName, "新建", "版本交付", fmt.Sprintf("%s-%s", args.EnvName, args.Version), string(bs), ctx.Logger) + + ctx.Err = deliveryservice.CreateHelmDeliveryVersion(args, ctx.Logger) +} + func DeleteDeliveryVersion(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - internalhandler.InsertOperationLog(c, ctx.Username, c.GetString("productName"), "删除", "版本交付", fmt.Sprintf("主键ID:%s", c.Param("id")), "", ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, c.GetString("productName"), "删除", "版本交付", c.Param("id"), "", ctx.Logger) //params validate ID := c.Param("id") @@ -382,13 +225,85 @@ func ListDeliveryServiceNames(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - productName := c.Query("productName") - orgIDStr := c.Query("orgId") - orgID, err := strconv.Atoi(orgIDStr) + productName := c.Query("projectName") + ctx.Resp, ctx.Err = deliveryservice.ListDeliveryServiceNames(productName, ctx.Logger) +} + +func DownloadDeliveryChart(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + versionName := c.Query("version") + chartName := c.Query("chartName") + projectName := c.Query("projectName") + + fileBytes, fileName, err := deliveryservice.DownloadDeliveryChart(projectName, versionName, chartName, ctx.Logger) if err != nil { - ctx.Err = e.ErrInvalidParam.AddDesc("orgId can't be empty!") + ctx.Err = err return } - ctx.Resp, ctx.Err = deliveryservice.ListDeliveryServiceNames(orgID, productName, ctx.Logger) + c.Writer.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, fileName)) + c.Data(http.StatusOK, "application/octet-stream", fileBytes) +} + +func GetChartVersionFromRepo(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + chartName := c.Query("chartName") + chartRepoName := c.Query("chartRepoName") + + ctx.Resp, ctx.Err = deliveryservice.GetChartVersion(chartName, chartRepoName) +} + +func PreviewGetDeliveryChart(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + versionName := c.Query("version") + chartName := c.Query("chartName") + projectName := c.Query("projectName") + + ctx.Resp, ctx.Err = deliveryservice.PreviewDeliveryChart(projectName, versionName, chartName, ctx.Logger) +} + +func GetDeliveryChartFilePath(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(deliveryservice.DeliveryChartFilePathArgs) + err := c.BindQuery(args) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + + ctx.Resp, ctx.Err = deliveryservice.GetDeliveryChartFilePath(args, ctx.Logger) +} + +func GetDeliveryChartFileContent(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(deliveryservice.DeliveryChartFileContentArgs) + err := c.BindQuery(args) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + ctx.Resp, ctx.Err = deliveryservice.GetDeliveryChartFileContent(args, ctx.Logger) +} + +func ApplyDeliveryGlobalVariables(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(deliveryservice.DeliveryVariablesApplyArgs) + err := c.BindJSON(args) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + ctx.Resp, ctx.Err = deliveryservice.ApplyDeliveryGlobalVariables(args, ctx.Logger) } diff --git a/pkg/microservice/aslan/core/delivery/service/artifact.go b/pkg/microservice/aslan/core/delivery/service/artifact.go index ac941e49b7b1598099c659280194f6c99c6e91a6..cf8ee9683319581599e2788cf978304a53b4a3d0 100644 --- a/pkg/microservice/aslan/core/delivery/service/artifact.go +++ b/pkg/microservice/aslan/core/delivery/service/artifact.go @@ -168,3 +168,12 @@ func getDeliveryActivitiesMap(deliveryActivities []*commonmodels.DeliveryActivit } return artifactMapInfo } + +func GetDeliveryArtifactIDByImage(deliveryArtifactArgs *commonrepo.DeliveryArtifactArgs, log *zap.SugaredLogger) (string, error) { + deliveryArtifact, err := commonrepo.NewDeliveryArtifactColl().Get(deliveryArtifactArgs) + if err != nil { + log.Errorf("get deliveryArtifact error: %s", err) + return "", e.ErrFindArtifact.AddErr(err) + } + return deliveryArtifact.ID.Hex(), nil +} diff --git a/pkg/microservice/aslan/core/delivery/service/product.go b/pkg/microservice/aslan/core/delivery/service/product.go index 9b2fdf8b7f595a815da10db2f46403dafa9de5e0..ece5c4adc91735b3c1c2050de5c8b30b7387f7ac 100644 --- a/pkg/microservice/aslan/core/delivery/service/product.go +++ b/pkg/microservice/aslan/core/delivery/service/product.go @@ -24,16 +24,6 @@ import ( e "github.com/koderover/zadig/pkg/tool/errors" ) -func FindDeliveryProduct(orgID int, log *zap.SugaredLogger) ([]string, error) { - productNames, err := commonrepo.NewDeliveryVersionColl().FindProducts(orgID) - if err != nil { - log.Errorf("find FindDeliveryProduct error: %v", err) - return []string{}, e.ErrFindDeliveryProducts - } - - return productNames, err -} - func GetProductByDeliveryInfo(username, releaseID string, log *zap.SugaredLogger) (*commonmodels.Product, error) { version := new(commonrepo.DeliveryVersionArgs) version.ID = releaseID diff --git a/pkg/microservice/aslan/core/delivery/service/version.go b/pkg/microservice/aslan/core/delivery/service/version.go index d9498b649189ab945fcbc316d40fc3be4f9a49c5..1c8f97b70c221700f6a690cf362dab5993a3aaed 100644 --- a/pkg/microservice/aslan/core/delivery/service/version.go +++ b/pkg/microservice/aslan/core/delivery/service/version.go @@ -17,21 +17,225 @@ limitations under the License. package service import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/blang/semver/v4" + "github.com/chartmuseum/helm-push/pkg/helm" + "github.com/hashicorp/go-multierror" + "github.com/otiai10/copy" + "github.com/pkg/errors" + "go.mongodb.org/mongo-driver/bson/primitive" "go.uber.org/zap" + chartloader "helm.sh/helm/v3/pkg/chart/loader" + "helm.sh/helm/v3/pkg/chartutil" + "helm.sh/helm/v3/pkg/repo" "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/yaml" + "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" + commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" + workflowservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" + "github.com/koderover/zadig/pkg/setting" e "github.com/koderover/zadig/pkg/tool/errors" + helmtool "github.com/koderover/zadig/pkg/tool/helmclient" + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types" + "github.com/koderover/zadig/pkg/util" + "github.com/koderover/zadig/pkg/util/converter" + fsutil "github.com/koderover/zadig/pkg/util/fs" + yamlutil "github.com/koderover/zadig/pkg/util/yaml" +) + +const ( + VerbosityBrief string = "brief" // brief delivery data + VerbosityDetailed string = "detailed" // detailed delivery version with total data ) +type DeliveryVersionFilter struct { + ServiceName string +} + +type CreateHelmDeliveryVersionOption struct { + EnableOfflineDist bool `json:"enableOfflineDist"` + S3StorageID string `json:"s3StorageID"` +} + +type ImageData struct { + ImageName string `json:"imageName"` + ImageTag string `json:"imageTag"` + Selected bool `json:"selected"` +} + +type CreateHelmDeliveryVersionChartData struct { + ServiceName string `json:"serviceName"` + Version string `json:"version,omitempty"` + ValuesYamlContent string `json:"valuesYamlContent"` + ImageData []*ImageData `json:"imageData"` +} + +type CreateHelmDeliveryVersionArgs struct { + CreateBy string `json:"-"` + ProductName string `json:"productName"` + Retry bool `json:"retry"` + Version string `json:"version"` + Desc string `json:"desc"` + EnvName string `json:"envName"` + Labels []string `json:"labels"` + ImageRepoName string `json:"imageRepoName"` + *DeliveryVersionChartData +} + +type DeliveryVersionChartData struct { + GlobalVariables string `json:"globalVariables"` + ChartRepoName string `json:"chartRepoName"` + ImageRegistryID string `json:"imageRegistryID"` + ChartDatas []*CreateHelmDeliveryVersionChartData `json:"chartDatas"` + Options *CreateHelmDeliveryVersionOption `json:"options"` +} + +type DeliveryChartData struct { + ChartData *CreateHelmDeliveryVersionChartData + ServiceObj *commonmodels.Service + ProductService *commonmodels.ProductService + RenderChart *template.RenderChart + RenderSet *commonmodels.RenderSet + ValuesInEnv map[string]interface{} +} + +type DeliveryChartResp struct { + FileInfos []*types.FileInfo `json:"fileInfos"` +} + +type DeliveryChartFilePathArgs struct { + Dir string `json:"dir"` + ProjectName string `json:"projectName"` + ChartName string `json:"chartName"` + Version string `json:"version"` +} + +type DeliveryChartFileContentArgs struct { + FilePath string `json:"filePath"` + FileName string `json:"fileName"` + ProjectName string `json:"projectName"` + ChartName string `json:"chartName"` + Version string `json:"version"` +} + +type DeliveryVariablesApplyArgs struct { + GlobalVariables string `json:"globalVariables,omitempty"` + ChartDatas []*CreateHelmDeliveryVersionChartData `json:"chartDatas"` +} + +type ListDeliveryVersionArgs struct { + Page int `form:"page"` + PerPage int `form:"per_page"` + TaskId int `form:"taskId"` + ServiceName string `form:"serviceName"` + Verbosity string `form:"verbosity"` + ProjectName string `form:"projectName"` + WorkflowName string `form:"workflowName"` +} + +type ReleaseInfo struct { + VersionInfo *commonmodels.DeliveryVersion `json:"versionInfo"` + BuildInfo []*commonmodels.DeliveryBuild `json:"buildInfo,omitempty"` + DeployInfo []*commonmodels.DeliveryDeploy `json:"deployInfo,omitempty"` + TestInfo []*commonmodels.DeliveryTest `json:"testInfo,omitempty"` + DistributeInfo []*commonmodels.DeliveryDistribute `json:"distributeInfo,omitempty"` + SecurityInfo []*DeliverySecurityStats `json:"securityStatsInfo,omitempty"` +} + +type DeliverySecurityStatsInfo struct { + Total int `json:"total"` + Unknown int `json:"unkown"` + Negligible int `json:"negligible"` + Low int `json:"low"` + Medium int `json:"medium"` + High int `json:"high"` + Critical int `json:"critical"` +} + +type DeliverySecurityStats struct { + ImageName string `json:"imageName"` + ImageID string `json:"imageId"` + DeliverySecurityStatsInfo DeliverySecurityStatsInfo `json:"deliverySecurityStatsInfo"` +} + +type ImageUrlDetail struct { + ImageUrl string + Name string + Registry string + Tag string + CustomTag string +} + +type ServiceImageDetails struct { + ServiceName string + Images []*ImageUrlDetail + Registries []string +} + +type ChartVersionResp struct { + ChartName string `json:"chartName"` + ChartVersion string `json:"chartVersion"` + NextChartVersion string `json:"nextChartVersion"` + Url string `json:"url"` +} + +type DeliveryVersionPayloadImage struct { + ServiceModule string `json:"service_module"` + Image string `json:"image"` +} + +type DeliveryVersionPayloadChart struct { + ChartName string `json:"chart_name"` + ChartVersion string `json:"chart_version"` + ChartUrl string `json:"chart_url"` + Images []*DeliveryVersionPayloadImage `json:"images"` +} + +type DeliveryVersionHookPayload struct { + ProjectName string `json:"project_name"` + Version string `json:"version"` + Status string `json:"status"` + Error string `json:"error"` + StartTime int64 `json:"start_time"` + EndTime int64 `json:"end_time"` + Charts []*DeliveryVersionPayloadChart `json:"charts"` +} + func GetDeliveryVersion(args *commonrepo.DeliveryVersionArgs, log *zap.SugaredLogger) (*commonmodels.DeliveryVersion, error) { - resp, err := commonrepo.NewDeliveryVersionColl().Get(args) + versionData, err := commonrepo.NewDeliveryVersionColl().Get(args) if err != nil { log.Errorf("get deliveryVersion error: %v", err) return nil, e.ErrGetDeliveryVersion } - return resp, err + return versionData, err +} + +func GetDetailReleaseData(args *commonrepo.DeliveryVersionArgs, log *zap.SugaredLogger) (*ReleaseInfo, error) { + versionData, err := commonrepo.NewDeliveryVersionColl().Get(args) + if err != nil { + log.Errorf("get deliveryVersion error: %v", err) + return nil, e.ErrGetDeliveryVersion + } + return buildListReleaseResp(VerbosityDetailed, versionData, nil, log) } func FindDeliveryVersion(args *commonrepo.DeliveryVersionArgs, log *zap.SugaredLogger) ([]*commonmodels.DeliveryVersion, error) { @@ -52,30 +256,1467 @@ func DeleteDeliveryVersion(args *commonrepo.DeliveryVersionArgs, log *zap.Sugare return nil } -func ListDeliveryServiceNames(orgID int, productName string, log *zap.SugaredLogger) ([]string, error) { - serviceNames := sets.String{} +func filterReleases(filter *DeliveryVersionFilter, deliveryVersion *commonmodels.DeliveryVersion, logger *zap.SugaredLogger) bool { + if filter == nil { + return true + } + if filter.ServiceName != "" { + deliveryDeployArgs := new(commonrepo.DeliveryDeployArgs) + deliveryDeployArgs.ReleaseID = deliveryVersion.ID.Hex() + deliveryDeploys, err := FindDeliveryDeploy(deliveryDeployArgs, logger) + if err != nil { + return true + } + match := false + for _, deliveryDeploy := range deliveryDeploys { + if deliveryDeploy.ServiceName == filter.ServiceName { + match = true + break + } + } + if !match { + return false + } + } + return true +} - version := new(commonrepo.DeliveryVersionArgs) - version.OrgID = orgID - version.ProductName = productName - deliveryVersions, err := FindDeliveryVersion(version, log) +func buildBriefRelease(deliveryVersion *commonmodels.DeliveryVersion, _ *zap.SugaredLogger) (*ReleaseInfo, error) { + return &ReleaseInfo{ + VersionInfo: deliveryVersion, + }, nil +} + +func buildDetailedRelease(deliveryVersion *commonmodels.DeliveryVersion, filterOpt *DeliveryVersionFilter, logger *zap.SugaredLogger) (*ReleaseInfo, error) { + releaseInfo := new(ReleaseInfo) + //versionInfo + releaseInfo.VersionInfo = deliveryVersion + + //deployInfo + deliveryDeployArgs := new(commonrepo.DeliveryDeployArgs) + deliveryDeployArgs.ReleaseID = deliveryVersion.ID.Hex() + deliveryDeploys, err := FindDeliveryDeploy(deliveryDeployArgs, logger) if err != nil { - log.Errorf("FindDeliveryVersion failed, err:%v", err) - return serviceNames.List(), err + return nil, err + } + if filterOpt != nil { + if !filterReleases(filterOpt, deliveryVersion, logger) { + return nil, nil + } + } + // 将serviceName替换为服务名/服务组件的形式，用于前端展示 + for _, deliveryDeploy := range deliveryDeploys { + if deliveryDeploy.ContainerName != "" { + deliveryDeploy.ServiceName = deliveryDeploy.ServiceName + "/" + deliveryDeploy.ContainerName + } + } + releaseInfo.DeployInfo = deliveryDeploys + + //buildInfo + deliveryBuildArgs := new(commonrepo.DeliveryBuildArgs) + deliveryBuildArgs.ReleaseID = deliveryVersion.ID.Hex() + deliveryBuilds, err := FindDeliveryBuild(deliveryBuildArgs, logger) + if err != nil { + return nil, err + } + releaseInfo.BuildInfo = deliveryBuilds + + //testInfo + deliveryTestArgs := new(commonrepo.DeliveryTestArgs) + deliveryTestArgs.ReleaseID = deliveryVersion.ID.Hex() + deliveryTests, err := FindDeliveryTest(deliveryTestArgs, logger) + if err != nil { + return nil, err + } + releaseInfo.TestInfo = deliveryTests + + //securityStatsInfo + deliverySecurityStatss := make([]*DeliverySecurityStats, 0) + if pipelineTask, err := workflowservice.GetPipelineTaskV2(int64(deliveryVersion.TaskID), deliveryVersion.WorkflowName, config.WorkflowType, logger); err == nil { + for _, subStage := range pipelineTask.Stages { + if subStage.TaskType == config.TaskSecurity { + subSecurityTaskMap := subStage.SubTasks + for _, subTask := range subSecurityTaskMap { + securityInfo, _ := base.ToSecurityTask(subTask) + + deliverySecurityStats := new(DeliverySecurityStats) + deliverySecurityStats.ImageName = securityInfo.ImageName + deliverySecurityStats.ImageID = securityInfo.ImageID + deliverySecurityStatsMap, err := FindDeliverySecurityStatistics(securityInfo.ImageID, logger) + if err != nil { + return nil, err + } + var transErr error + b, err := json.Marshal(deliverySecurityStatsMap) + if err != nil { + transErr = fmt.Errorf("marshal task error: %v", err) + } + if err := json.Unmarshal(b, &deliverySecurityStats.DeliverySecurityStatsInfo); err != nil { + transErr = fmt.Errorf("unmarshal task error: %v", err) + } + if transErr != nil { + return nil, transErr + } + + deliverySecurityStatss = append(deliverySecurityStatss, deliverySecurityStats) + } + break + } + } + releaseInfo.SecurityInfo = deliverySecurityStatss + } + + //distributeInfo + deliveryDistributeArgs := new(commonrepo.DeliveryDistributeArgs) + deliveryDistributeArgs.ReleaseID = deliveryVersion.ID.Hex() + deliveryDistributes, _ := FindDeliveryDistribute(deliveryDistributeArgs, logger) + releaseInfo.DistributeInfo = deliveryDistributes + + // fill some data for helm delivery releases + processReleaseRespData(releaseInfo) + + return releaseInfo, nil +} + +func buildListReleaseResp(verbosity string, deliveryVersion *commonmodels.DeliveryVersion, filterOpt *DeliveryVersionFilter, logger *zap.SugaredLogger) (*ReleaseInfo, error) { + switch verbosity { + case VerbosityBrief: + return buildBriefRelease(deliveryVersion, logger) + case VerbosityDetailed: + return buildDetailedRelease(deliveryVersion, filterOpt, logger) + default: + return buildDetailedRelease(deliveryVersion, filterOpt, logger) + } +} + +func ListDeliveryVersion(args *ListDeliveryVersionArgs, logger *zap.SugaredLogger) ([]*ReleaseInfo, error) { + versionListArgs := new(commonrepo.DeliveryVersionArgs) + versionListArgs.ProductName = args.ProjectName + versionListArgs.WorkflowName = args.WorkflowName + versionListArgs.TaskID = args.TaskId + versionListArgs.PerPage = args.PerPage + versionListArgs.Page = args.Page + deliveryVersions, err := FindDeliveryVersion(versionListArgs, logger) + if err != nil { + return nil, err } + releaseInfos := make([]*ReleaseInfo, 0) for _, deliveryVersion := range deliveryVersions { - deliveryDeployArgs := new(commonrepo.DeliveryDeployArgs) - deliveryDeployArgs.ReleaseID = deliveryVersion.ID.Hex() - deliveryDeploys, err := FindDeliveryDeploy(deliveryDeployArgs, log) + releaseInfo, err := buildListReleaseResp(args.Verbosity, deliveryVersion, &DeliveryVersionFilter{ServiceName: args.ServiceName}, logger) if err != nil { - log.Errorf("FindDeliveryDeploy failed, ReleaseID:%s, err:%v", deliveryVersion.ID, err) + return nil, err + } + if releaseInfo == nil { continue } - for _, deliveryDeploy := range deliveryDeploys { - serviceNames.Insert(deliveryDeploy.ServiceName) + releaseInfos = append(releaseInfos, releaseInfo) + } + return releaseInfos, nil +} + +// fill release +func processReleaseRespData(release *ReleaseInfo) { + if release.VersionInfo.Type != setting.DeliveryVersionTypeChart { + return + } + + distributeImageMap := make(map[string][]*commonmodels.DeliveryDistribute) + for _, distributeImage := range release.DistributeInfo { + if distributeImage.DistributeType != config.Image { + continue } + distributeImageMap[distributeImage.ChartName] = append(distributeImageMap[distributeImage.ChartName], distributeImage) } - return serviceNames.UnsortedList(), nil + chartDistributeCount := 0 + distributes := make([]*commonmodels.DeliveryDistribute, 0) + for _, distribute := range release.DistributeInfo { + if distribute.DistributeType == config.Image { + continue + } + switch distribute.DistributeType { + case config.Chart: + chartDistributeCount++ + distribute.SubDistributes = distributeImageMap[distribute.ChartName] + case config.File: + s3Storage, err := commonrepo.NewS3StorageColl().Find(distribute.S3StorageID) + if err != nil { + log.Errorf("failed to query s3 storageID: %s, err: %s", distribute.S3StorageID, err) + } else { + distribute.StorageURL = s3Storage.Endpoint + distribute.StorageBucket = s3Storage.Bucket + } + } + distributes = append(distributes, distribute) + } + release.DistributeInfo = distributes + + release.VersionInfo.Progress = buildDeliveryProgressInfo(release.VersionInfo, chartDistributeCount) +} + +func buildDeliveryProgressInfo(deliveryVersion *commonmodels.DeliveryVersion, successfulChartCount int) *commonmodels.DeliveryVersionProgress { + if deliveryVersion.Type != setting.DeliveryVersionTypeChart { + return nil + } + + _, err := checkVersionStatus(deliveryVersion) + if err != nil { + updateVersionStatus(deliveryVersion.Version, deliveryVersion.ProductName, setting.DeliveryVersionStatusFailed, err.Error()) + } + + progress := &commonmodels.DeliveryVersionProgress{ + SuccessChartCount: successfulChartCount, + TotalChartCount: 0, + PackageUploadStatus: "", + Error: "", + } + if deliveryVersion.Status == setting.DeliveryVersionStatusSuccess { + progress.TotalChartCount = successfulChartCount + progress.PackageUploadStatus = setting.DeliveryVersionPackageStatusSuccess + return progress + } + + argsBytes, err := json.Marshal(deliveryVersion.CreateArgument) + if err != nil { + log.Errorf("failed to marshal arguments, versionName: %s err %s", deliveryVersion.Version, err) + return progress + } + createArgs := new(DeliveryVersionChartData) + err = json.Unmarshal(argsBytes, createArgs) + if err != nil { + log.Errorf("failed to unMarshal arguments, versionName: %s err %s", deliveryVersion.Version, err) + return progress + } + + progress.TotalChartCount = len(createArgs.ChartDatas) + + if deliveryVersion.Status == setting.DeliveryVersionStatusFailed { + progress.PackageUploadStatus = setting.DeliveryVersionPackageStatusFailed + progress.Error = deliveryVersion.Error + return progress + } + + if len(createArgs.ChartDatas) > successfulChartCount { + progress.PackageUploadStatus = setting.DeliveryVersionPackageStatusWaiting + return progress + } + + progress.PackageUploadStatus = setting.DeliveryVersionPackageStatusUploading + return progress + +} + +func getChartTGZDir(productName, versionName string) string { + tmpDir := os.TempDir() + return filepath.Join(tmpDir, "chart-tgz", productName, versionName) +} + +func getChartExpandDir(productName, versionName string) string { + tmpDir := os.TempDir() + return filepath.Join(tmpDir, "chart", productName, versionName) +} + +func getProductEnvInfo(productName, envName string) (*commonmodels.Product, error) { + productInfo, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: productName, + EnvName: envName, + }) + if err != nil { + log.Errorf("failed to query product info, productName: %s envName: %s err: %s", productName, envName, err) + return nil, fmt.Errorf("failed to query product info, productName: %s envName: %s", productName, envName) + } + return productInfo, nil +} + +func getChartRepoData(repoName string) (*commonmodels.HelmRepo, error) { + return commonrepo.NewHelmRepoColl().Find(&commonrepo.HelmRepoFindOption{RepoName: repoName}) +} + +// ensure chart files exist +func ensureChartFiles(chartData *DeliveryChartData, prod *commonmodels.Product) (string, error) { + serviceObj := chartData.ServiceObj + revisionBasePath := config.LocalDeliveryChartPathWithRevision(serviceObj.ProductName, serviceObj.ServiceName, serviceObj.Revision) + deliveryChartPath := filepath.Join(revisionBasePath, serviceObj.ServiceName) + if exists, _ := fsutil.DirExists(deliveryChartPath); exists { + return deliveryChartPath, nil + } + + serviceName, revision := serviceObj.ServiceName, serviceObj.Revision + basePath := config.LocalServicePathWithRevision(serviceObj.ProductName, serviceName, revision) + if err := commonservice.PreloadServiceManifestsByRevision(basePath, serviceObj); err != nil { + log.Warnf("failed to get chart of revision: %d for service: %s, use latest version", revision, serviceName) + // use the latest version when it fails to download the specific version + basePath = config.LocalServicePath(serviceObj.ProductName, serviceName) + if err = commonservice.PreLoadServiceManifests(basePath, serviceObj); err != nil { + log.Errorf("failed to load chart info for service %v", serviceObj.ServiceName) + return "", err + } + } + + fullPath := filepath.Join(basePath, serviceObj.ServiceName) + err := copy.Copy(fullPath, deliveryChartPath) + if err != nil { + return "", err + } + + restConfig, err := kube.GetRESTConfig(prod.ClusterID) + if err != nil { + log.Errorf("get rest config error: %s", err) + return "", err + } + helmClient, err := helmtool.NewClientFromRestConf(restConfig, prod.Namespace) + if err != nil { + log.Errorf("[%s][%s] init helm client error: %s", prod.ProductName, prod.Namespace, err) + return "", err + } + + releaseName := util.GeneReleaseName(serviceObj.GetReleaseNaming(), prod.ProductName, prod.Namespace, prod.EnvName, serviceObj.ServiceName) + valuesMap, err := helmClient.GetReleaseValues(releaseName, true) + if err != nil { + log.Errorf("failed to get values map data, err: %s", err) + return "", err + } + + currentValuesYaml, err := yaml.Marshal(valuesMap) + if err != nil { + return "", err + } + + err = os.WriteFile(filepath.Join(deliveryChartPath, setting.ValuesYaml), currentValuesYaml, 0644) + if err != nil { + return "", errors.Wrapf(err, "failed to write values.yaml") + } + + return deliveryChartPath, nil +} + +// handleImageRegistry update image registry to target registry +func handleImageRegistry(valuesYaml []byte, chartData *DeliveryChartData, targetRegistry *commonmodels.RegistryNamespace, + registryMap map[string]*commonmodels.RegistryNamespace, imageData []*ImageData) ([]byte, *ServiceImageDetails, error) { + + flatMap, err := converter.YamlToFlatMap(valuesYaml) + if err != nil { + return nil, nil, err + } + + imageTagMap := make(map[string]string) + for _, it := range imageData { + if it.Selected { + imageTagMap[it.ImageName] = it.ImageTag + } + } + + retValuesYaml := string(valuesYaml) + + serviceObj := chartData.ServiceObj + imagePathSpecs := make([]map[string]string, 0) + for _, container := range serviceObj.Containers { + imageSearchRule := &template.ImageSearchingRule{ + Repo: container.ImagePath.Repo, + Image: container.ImagePath.Image, + Tag: container.ImagePath.Tag, + } + pattern := imageSearchRule.GetSearchingPattern() + imagePathSpecs = append(imagePathSpecs, pattern) + } + + imageDetail := &ServiceImageDetails{ + ServiceName: serviceObj.ServiceName, + Images: make([]*ImageUrlDetail, 0), + } + + registrySet := sets.NewString() + for _, spec := range imagePathSpecs { + imageUrl, err := commonservice.GeneImageURI(spec, flatMap) + if err != nil { + return nil, nil, err + } + + imageName := commonservice.ExtractImageName(imageUrl) + imageTag := commonservice.ExtractImageTag(imageUrl) + + customTag := "" + if ct, ok := imageTagMap[imageName]; ok { + customTag = ct + } else { + continue + } + + if customTag == "" { + customTag = imageTag + } + + registryUrl, err := commonservice.ExtractImageRegistry(imageUrl) + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to parse registry from image uri: %s", imageUrl) + } + registryUrl = strings.TrimSuffix(registryUrl, "/") + + registryID := "" + // used source registry + if registry, ok := registryMap[registryUrl]; ok { + registryID = registry.ID.Hex() + registrySet.Insert(registryID) + } + + imageDetail.Images = append(imageDetail.Images, &ImageUrlDetail{ + ImageUrl: imageUrl, + Name: imageName, + Tag: imageTag, + Registry: registryID, + CustomTag: customTag, + }) + + // assign image to values.yaml + targetImageUrl := util.ReplaceRepo(imageUrl, targetRegistry.RegAddr, targetRegistry.Namespace) + targetImageUrl = util.ReplaceTag(targetImageUrl, customTag) + replaceValuesMap, err := commonservice.AssignImageData(targetImageUrl, spec) + if err != nil { + return nil, nil, fmt.Errorf("failed to pase image uri %s, err %s", targetImageUrl, err) + } + + // replace image into final merged values.yaml + retValuesYaml, err = commonservice.ReplaceImage(retValuesYaml, replaceValuesMap) + if err != nil { + return nil, nil, err + } + } + + imageDetail.Registries = registrySet.List() + return []byte(retValuesYaml), imageDetail, nil +} + +func handleSingleChart(chartData *DeliveryChartData, product *commonmodels.Product, chartRepo *commonmodels.HelmRepo, dir string, globalVariables string, + targetRegistry *commonmodels.RegistryNamespace, registryMap map[string]*commonmodels.RegistryNamespace) (*ServiceImageDetails, error) { + serviceObj := chartData.ServiceObj + + deliveryChartPath, err := ensureChartFiles(chartData, product) + if err != nil { + return nil, err + } + + valuesYamlData := make(map[string]interface{}) + valuesFilePath := filepath.Join(deliveryChartPath, setting.ValuesYaml) + valueYamlContent, err := os.ReadFile(valuesFilePath) + if err != nil { + return nil, errors.Wrapf(err, "failed to read values.yaml for service %s", serviceObj.ServiceName) + } + + // write values.yaml file before load + if len(chartData.ChartData.ValuesYamlContent) > 0 { // values.yaml was edited directly + if err = yaml.Unmarshal([]byte(chartData.ChartData.ValuesYamlContent), map[string]interface{}{}); err != nil { + log.Errorf("invalid yaml content, serviceName: %s, yamlContent: %s", serviceObj.ServiceName, chartData.ChartData.ValuesYamlContent) + return nil, errors.Wrapf(err, "invalid yaml content for service: %s", serviceObj.ServiceName) + } + valueYamlContent = []byte(chartData.ChartData.ValuesYamlContent) + } else if len(globalVariables) > 0 { // merge global variables + valueYamlContent, err = yamlutil.Merge([][]byte{valueYamlContent, []byte(globalVariables)}) + if err != nil { + return nil, errors.Wrapf(err, "failed to merge global variables for service: %s", serviceObj.ServiceName) + } + } + + // replace image url(registryUrl and imageTag) + valueYamlContent, imageDetail, err := handleImageRegistry(valueYamlContent, chartData, targetRegistry, registryMap, chartData.ChartData.ImageData) + if err != nil { + return nil, errors.Wrapf(err, "failed to handle image registry for service: %s", serviceObj.ServiceName) + } + + err = yaml.Unmarshal(valueYamlContent, &valuesYamlData) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal values.yaml for service %s", serviceObj.ServiceName) + } + + // hold the currently running yaml data + chartData.ValuesInEnv = valuesYamlData + + err = os.WriteFile(valuesFilePath, valueYamlContent, 0644) + if err != nil { + return nil, errors.Wrapf(err, "failed to write values.yaml file for service %s", serviceObj.ServiceName) + } + + //load chart info from local storage + chartRequested, err := chartloader.Load(deliveryChartPath) + if err != nil { + return nil, errors.Wrapf(err, "failed to load chart info, path %s", deliveryChartPath) + } + + //set metadata + chartRequested.Metadata.Name = chartData.ChartData.ServiceName + chartRequested.Metadata.Version = chartData.ChartData.Version + chartRequested.Metadata.AppVersion = chartData.ChartData.Version + + //create local chart package + chartPackagePath, err := helm.CreateChartPackage(&helm.Chart{Chart: chartRequested}, dir) + if err != nil { + return nil, err + } + + client, err := helmtool.NewClient() + if err != nil { + return nil, errors.Wrapf(err, "failed to create chart repo client, repoName: %s", chartRepo.RepoName) + } + + log.Infof("pushing chart %s to %s...", filepath.Base(chartPackagePath), chartRepo.URL) + err = client.PushChart(commonservice.GeneHelmRepo(chartRepo), chartPackagePath) + if err != nil { + return nil, errors.Wrapf(err, "failed to push chart: %s", chartPackagePath) + } + return imageDetail, nil +} + +func makeChartTGZFileDir(productName, versionName string) (string, error) { + dirPath := getChartTGZDir(productName, versionName) + if err := os.RemoveAll(dirPath); err != nil { + if !os.IsExist(err) { + return "", errors.Wrapf(err, "failed to claer dir for chart tgz files") + } + } + err := os.MkdirAll(dirPath, 0777) + if err != nil { + return "", errors.Wrapf(err, "failed to create chart tgz dir for version: %s", versionName) + } + return dirPath, nil +} + +func CreateHelmDeliveryVersion(args *CreateHelmDeliveryVersionArgs, logger *zap.SugaredLogger) error { + if args.Retry { + return RetryCreateHelmDeliveryVersion(args.ProductName, args.Version, logger) + } else { + return CreateNewHelmDeliveryVersion(args, logger) + } +} + +// validate chartInfo, make sure service is in environment +// prepare data set for chart delivery +func prepareChartData(chartDatas []*CreateHelmDeliveryVersionChartData, productInfo *commonmodels.Product) (map[string]*DeliveryChartData, error) { + + renderSet, err := commonrepo.NewRenderSetColl().Find(&commonrepo.RenderSetFindOption{ + Revision: productInfo.Render.Revision, + Name: productInfo.Render.Name, + EnvName: productInfo.EnvName, + ProductTmpl: productInfo.ProductName, + }) + if err != nil { + return nil, fmt.Errorf("failed to find renderSet: %s, revision: %d", productInfo.Render.Name, productInfo.Render.Revision) + } + chartMap := make(map[string]*template.RenderChart) + for _, rChart := range renderSet.ChartInfos { + chartMap[rChart.ServiceName] = rChart + } + + chartDataMap := make(map[string]*DeliveryChartData) + serviceMap := productInfo.GetServiceMap() + for _, chartData := range chartDatas { + if productService, ok := serviceMap[chartData.ServiceName]; ok { + serviceObj, err := commonrepo.NewServiceColl().Find(&commonrepo.ServiceFindOption{ + ServiceName: chartData.ServiceName, + Revision: productService.Revision, + Type: setting.HelmDeployType, + ProductName: productInfo.ProductName, + }) + if err != nil { + return nil, fmt.Errorf("failed to query service: %s", chartData.ServiceName) + } + renderChart, ok := chartMap[chartData.ServiceName] + if !ok { + return nil, fmt.Errorf("can't find renderChart for service: %s", chartData.ServiceName) + } + chartDataMap[chartData.ServiceName] = &DeliveryChartData{ + ChartData: chartData, + RenderChart: renderChart, + ServiceObj: serviceObj, + ProductService: productService, + RenderSet: renderSet, + } + } else { + return nil, fmt.Errorf("service %s not found in environment", chartData.ServiceName) + } + } + return chartDataMap, nil +} + +func buildRegistryMap() (map[string]*commonmodels.RegistryNamespace, error) { + registries, err := commonservice.ListRegistryNamespaces("", true, log.SugaredLogger()) + if err != nil { + return nil, fmt.Errorf("failed to query registries") + } + ret := make(map[string]*commonmodels.RegistryNamespace) + for _, singleRegistry := range registries { + fullUrl := fmt.Sprintf("%s/%s", singleRegistry.RegAddr, singleRegistry.Namespace) + fullUrl = strings.TrimSuffix(fullUrl, "/") + u, _ := url.Parse(fullUrl) + if len(u.Scheme) > 0 { + fullUrl = strings.TrimPrefix(fullUrl, fmt.Sprintf("%s://", u.Scheme)) + } + ret[fullUrl] = singleRegistry + } + return ret, nil +} + +func buildArtifactTaskArgs(projectName, envName string, imagesMap *sync.Map) *commonmodels.ArtifactPackageTaskArgs { + imageArgs := make([]*commonmodels.ImagesByService, 0) + sourceRegistry := sets.NewString() + imagesMap.Range(func(key, value interface{}) bool { + imageDetail := value.(*ServiceImageDetails) + imagesByService := &commonmodels.ImagesByService{ + ServiceName: imageDetail.ServiceName, + Images: make([]*commonmodels.ImageData, 0), + } + for _, image := range imageDetail.Images { + imagesByService.Images = append(imagesByService.Images, &commonmodels.ImageData{ + ImageUrl: image.ImageUrl, + ImageName: image.Name, + ImageTag: image.Tag, + CustomTag: image.CustomTag, + RegistryID: image.Registry, + }) + } + imageArgs = append(imageArgs, imagesByService) + sourceRegistry.Insert(imageDetail.Registries...) + return true + }) + + ret := &commonmodels.ArtifactPackageTaskArgs{ + ProjectName: projectName, + EnvName: envName, + Images: imageArgs, + SourceRegistries: sourceRegistry.List(), + } + return ret +} + +// insert delivery distribution data for single chart, include image and chart +func insertDeliveryDistributions(result *task.ServicePackageResult, chartVersion string, deliveryVersion *commonmodels.DeliveryVersion, args *DeliveryVersionChartData) error { + for _, image := range result.ImageData { + err := commonrepo.NewDeliveryDistributeColl().Insert(&commonmodels.DeliveryDistribute{ + ReleaseID: deliveryVersion.ID, + ServiceName: image.ImageName, // image name + ChartName: result.ServiceName, + DistributeType: config.Image, + RegistryName: image.ImageUrl, + Namespace: commonservice.ExtractRegistryNamespace(image.ImageUrl), + CreatedAt: time.Now().Unix(), + }) + if err != nil { + log.Errorf("failed to insert image distribute data, chartName: %s, err: %s", result.ServiceName, err) + return fmt.Errorf("failed to insert image distribute data, chartName: %s", result.ServiceName) + } + } + + err := commonrepo.NewDeliveryDistributeColl().Insert(&commonmodels.DeliveryDistribute{ + ReleaseID: deliveryVersion.ID, + DistributeType: config.Chart, + ChartName: result.ServiceName, + ChartVersion: chartVersion, + ChartRepoName: args.ChartRepoName, + SubDistributes: nil, + CreatedAt: time.Now().Unix(), + }) + if err != nil { + log.Errorf("failed to insert chart distribute data, chartName: %s, err: %s", result.ServiceName, err) + return fmt.Errorf("failed to insert chart distribute data, chartName: %s", result.ServiceName) + } + return nil +} + +func buildDeliveryCharts(chartDataMap map[string]*DeliveryChartData, deliveryVersion *commonmodels.DeliveryVersion, args *DeliveryVersionChartData, logger *zap.SugaredLogger) (err error) { + defer func() { + if err != nil { + deliveryVersion.Status = setting.DeliveryVersionStatusFailed + deliveryVersion.Error = err.Error() + } + updateVersionStatus(deliveryVersion.Version, deliveryVersion.ProductName, deliveryVersion.Status, deliveryVersion.Error) + }() + + var errLock sync.Mutex + errorList := &multierror.Error{} + + appendError := func(err error) { + errLock.Lock() + defer errLock.Unlock() + errorList = multierror.Append(errorList, err) + } + + dir, err := makeChartTGZFileDir(deliveryVersion.ProductName, deliveryVersion.Version) + if err != nil { + return err + } + repoInfo, err := getChartRepoData(args.ChartRepoName) + if err != nil { + log.Errorf("failed to query chart-repo info, productName: %s, err: %s", deliveryVersion.ProductName, err) + return fmt.Errorf("failed to query chart-repo info, productName: %s, repoName: %s", deliveryVersion.ProductName, args.ChartRepoName) + } + + registryMap, err := buildRegistryMap() + if err != nil { + return fmt.Errorf("failed to build registry map") + } + + var targetRegistry *commonmodels.RegistryNamespace + for _, registry := range registryMap { + if registry.ID.Hex() == args.ImageRegistryID { + targetRegistry = registry + break + } + } + + imagesDataMap := &sync.Map{} + + wg := sync.WaitGroup{} + for _, chartData := range chartDataMap { + wg.Add(1) + go func(cData *DeliveryChartData) { + defer wg.Done() + // generate new chart data, push to chart repo, extract related images + imageData, err := handleSingleChart(cData, deliveryVersion.ProductEnvInfo, repoInfo, dir, args.GlobalVariables, targetRegistry, registryMap) + if err != nil { + logger.Errorf("failed to build chart package, serviceName: %s err: %s", cData.ChartData.ServiceName, err) + appendError(err) + return + } + imagesDataMap.Store(cData.ServiceObj.ServiceName, imageData) + }(chartData) + } + wg.Wait() + + if errorList.ErrorOrNil() != nil { + err = errorList.ErrorOrNil() + return + } + + // create task to deal with images + // offline docker images are not supported + taskArgs := buildArtifactTaskArgs(deliveryVersion.ProductName, deliveryVersion.ProductEnvInfo.EnvName, imagesDataMap) + taskArgs.TargetRegistries = []string{args.ImageRegistryID} + taskID, err := workflowservice.CreateArtifactPackageTask(taskArgs, deliveryVersion.Version, logger) + if err != nil { + return err + } + deliveryVersion.TaskID = int(taskID) + err = commonrepo.NewDeliveryVersionColl().UpdateTaskID(deliveryVersion.Version, deliveryVersion.ProductName, int32(deliveryVersion.TaskID)) + if err != nil { + logger.Errorf("failed to update delivery version task_id, version: %s, task_id: %s, err: %s", deliveryVersion, deliveryVersion.ProductName, deliveryVersion.TaskID) + } + // start a new routine to check task results + go waitVersionDone(deliveryVersion) + + return +} + +// send hook +func sendVersionDeliveryHook(deliveryVersion *commonmodels.DeliveryVersion, host, urlPath string) error { + projectName, version := deliveryVersion.ProductName, deliveryVersion.Version + ret := &DeliveryVersionHookPayload{ + ProjectName: projectName, + Version: version, + Status: setting.DeliveryVersionStatusSuccess, + Error: "", + StartTime: deliveryVersion.CreatedAt, + EndTime: time.Now().Unix(), + Charts: make([]*DeliveryVersionPayloadChart, 0), + } + + //distributes image + chart + deliveryDistributes, err := FindDeliveryDistribute(&commonrepo.DeliveryDistributeArgs{ + ReleaseID: deliveryVersion.ID.Hex(), + }, log.SugaredLogger()) + if err != nil { + return err + } + + distributeImageMap := make(map[string][]*DeliveryVersionPayloadImage) + for _, distributeImage := range deliveryDistributes { + if distributeImage.DistributeType != config.Image { + continue + } + distributeImageMap[distributeImage.ChartName] = append(distributeImageMap[distributeImage.ChartName], &DeliveryVersionPayloadImage{ + ServiceModule: distributeImage.ServiceName, + Image: distributeImage.RegistryName, + }) + } + + chartRepoName := "" + for _, distribute := range deliveryDistributes { + if distribute.DistributeType != config.Chart { + continue + } + ret.Charts = append(ret.Charts, &DeliveryVersionPayloadChart{ + ChartName: distribute.ChartName, + ChartVersion: distribute.ChartVersion, + Images: distributeImageMap[distribute.ChartName], + }) + chartRepoName = distribute.ChartRepoName + } + err = fillChartUrl(ret.Charts, chartRepoName) + if err != nil { + return err + } + + targetPath := fmt.Sprintf("%s/%s", host, strings.TrimPrefix(urlPath, "/")) + + // validate url + _, err = url.Parse(targetPath) + if err != nil { + return err + } + + reqBody, err := json.Marshal(ret) + if err != nil { + log.Errorf("marshal json args error: %s", err) + return err + } + + request, err := http.NewRequest("POST", targetPath, bytes.NewBuffer(reqBody)) + if err != nil { + return err + } + request.Header.Set("Content-Type", "application/json") + + client := http.Client{} + resp, err := client.Do(request) + if err != nil { + return err + } + + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("hook request send to url: %s got error resp code : %d", targetPath, resp.StatusCode) + } + + return nil +} + +func updateVersionStatus(versionName, projectName, status, errStr string) { + // send hook info when version build finished + if status == setting.DeliveryVersionStatusSuccess { + + versionInfo, err := commonrepo.NewDeliveryVersionColl().Get(&commonrepo.DeliveryVersionArgs{ + ProductName: projectName, + Version: versionName, + }) + if err != nil { + log.Errorf("failed to find version: %s of project %s, err: %s", versionName, projectName, err) + return + } + if versionInfo.Status == status { + return + } + + templateProduct, err := templaterepo.NewProductColl().Find(projectName) + if err != nil { + log.Errorf("updateVersionStatus failed to find template product: %s, err: %s", projectName, err) + } else { + hookConfig := templateProduct.DeliveryVersionHook + if hookConfig != nil && hookConfig.Enable { + err = sendVersionDeliveryHook(versionInfo, hookConfig.HookHost, hookConfig.Path) + if err != nil { + log.Errorf("updateVersionStatus failed to send version delivery hook, projectName: %s, err: %s", projectName, err) + } + } + } + } + + err := commonrepo.NewDeliveryVersionColl().UpdateStatusByName(versionName, projectName, status, errStr) + if err != nil { + log.Errorf("failed to update version status, name: %s, err: %s", versionName, err) + } +} + +func taskFinished(status config.Status) bool { + return status == config.StatusPassed || status == config.StatusFailed || status == config.StatusTimeout || status == config.StatusCancelled +} + +func waitVersionDone(deliveryVersion *commonmodels.DeliveryVersion) { + waitTimeout := time.After(60 * time.Minute * 1) + for { + select { + case <-waitTimeout: + updateVersionStatus(deliveryVersion.Version, deliveryVersion.ProductName, setting.DeliveryVersionStatusFailed, "timeout") + return + default: + done, err := checkVersionStatus(deliveryVersion) + if err != nil { + updateVersionStatus(deliveryVersion.Version, deliveryVersion.ProductName, setting.DeliveryVersionStatusFailed, err.Error()) + return + } + if done { + return + } + } + time.Sleep(time.Second * 5) + } +} + +func checkVersionStatus(deliveryVersion *commonmodels.DeliveryVersion) (bool, error) { + if deliveryVersion.Status == setting.DeliveryVersionStatusSuccess || deliveryVersion.Status == setting.DeliveryVersionStatusFailed { + return true, nil + } + pipelineName := fmt.Sprintf("%s-%s-%s", deliveryVersion.ProductName, deliveryVersion.ProductEnvInfo.EnvName, "artifact") + taskData, err := commonrepo.NewTaskColl().Find(int64(deliveryVersion.TaskID), pipelineName, config.ArtifactType) + if err != nil { + return false, fmt.Errorf("failed to query taskData, id: %d, pipelineName: %s", deliveryVersion.TaskID, pipelineName) + } + + if len(taskData.Stages) != 1 { + return false, fmt.Errorf("invalid task data, stage length not leagal") + } + + argsBytes, err := json.Marshal(deliveryVersion.CreateArgument) + if err != nil { + return false, errors.Wrapf(err, "failed to marshal arguments, versionName: %s err: %s", deliveryVersion.Version, err) + } + createArgs := new(DeliveryVersionChartData) + err = json.Unmarshal(argsBytes, createArgs) + if err != nil { + return false, errors.Wrapf(err, "failed to unMarshal arguments, versionName: %s err: %s", deliveryVersion.Version, err) + } + + distributes, err := commonrepo.NewDeliveryDistributeColl().Find(&commonrepo.DeliveryDistributeArgs{ + DistributeType: config.Chart, + ReleaseID: deliveryVersion.ID.Hex(), + }) + if err != nil { + return false, errors.Wrapf(err, "failed to query distrubutes, versionName: %s", deliveryVersion.Version) + } + + // for charts has been successfully handled, download charts directly + successCharts := sets.NewString() + for _, distribute := range distributes { + successCharts.Insert(distribute.ChartName) + } + + stage := taskData.Stages[0] + errorList := &multierror.Error{} + + allTaskDone := true + for _, taskData := range stage.SubTasks { + artifactPackageArgs, err := base.ToArtifactPackageImageTask(taskData) + if err != nil { + return false, errors.Wrapf(err, "failed to generate origin artifact task data") + } + + progressData, err := artifactPackageArgs.GetProgress() + if err != nil { + return false, errors.Wrapf(err, "failed to get progress data from data") + } + progressDataMap := make(map[string]*task.ServicePackageResult) + for _, singleResult := range progressData { + progressDataMap[singleResult.ServiceName] = singleResult + } + + for _, chartData := range createArgs.ChartDatas { + // service artifact has been marked as success + if successCharts.Has(chartData.ServiceName) { + continue + } + if singleResult, ok := progressDataMap[chartData.ServiceName]; ok { + if singleResult.Result != "success" { + errorList = multierror.Append(errorList, fmt.Errorf("failed to build image distribute for service:%s, err: %s ", singleResult.ServiceName, singleResult.ErrorMsg)) + continue + } + err = insertDeliveryDistributions(singleResult, chartData.Version, deliveryVersion, createArgs) + if err != nil { + errorList = multierror.Append(errorList, fmt.Errorf("failed to insert distribute data for service:%s ", singleResult.ServiceName)) + continue + } + successCharts.Insert(chartData.ServiceName) + } + } + + if !taskFinished(artifactPackageArgs.TaskStatus) { + allTaskDone = false + } + if len(artifactPackageArgs.Error) > 0 { + errorList = multierror.Append(errorList, fmt.Errorf(artifactPackageArgs.Error)) + } + } + + if allTaskDone { + if successCharts.Len() == len(createArgs.ChartDatas) { + deliveryVersion.Status = setting.DeliveryVersionStatusSuccess + } else { + deliveryVersion.Status = setting.DeliveryVersionStatusFailed + } + } + if errorList.ErrorOrNil() != nil { + deliveryVersion.Error = errorList.Error() + } + updateVersionStatus(deliveryVersion.Version, deliveryVersion.ProductName, deliveryVersion.Status, deliveryVersion.Error) + return allTaskDone, nil +} + +func CreateNewHelmDeliveryVersion(args *CreateHelmDeliveryVersionArgs, logger *zap.SugaredLogger) error { + // need appoint chart info + if len(args.ChartDatas) == 0 { + return e.ErrCreateDeliveryVersion.AddDesc("no chart info appointed") + } + // validate necessary params + if len(args.ChartRepoName) == 0 { + return e.ErrCreateDeliveryVersion.AddDesc("chart repo not appointed") + } + if len(args.ImageRegistryID) == 0 { + return e.ErrCreateDeliveryVersion.AddDesc("image registry not appointed") + } + // prepare data + productInfo, err := getProductEnvInfo(args.ProductName, args.EnvName) + if err != nil { + log.Infof("failed to query product info, productName: %s envName %s, err: %s", args.ProductName, args.EnvName, err) + return e.ErrCreateDeliveryVersion.AddDesc(fmt.Sprintf("failed to query product info, procutName: %s envName %s", args.ProductName, args.EnvName)) + } + chartDataMap, err := prepareChartData(args.ChartDatas, productInfo) + if err != nil { + return e.ErrCreateDeliveryVersion.AddErr(err) + } + + productInfo.ID, _ = primitive.ObjectIDFromHex("") + + versionObj := &commonmodels.DeliveryVersion{ + Version: args.Version, + ProductName: args.ProductName, + Type: setting.DeliveryVersionTypeChart, + Desc: args.Desc, + Labels: args.Labels, + ProductEnvInfo: productInfo, + Status: setting.DeliveryVersionStatusCreating, + CreateArgument: args.DeliveryVersionChartData, + CreatedBy: args.CreateBy, + CreatedAt: time.Now().Unix(), + DeletedAt: 0, + } + + err = commonrepo.NewDeliveryVersionColl().Insert(versionObj) + if err != nil { + logger.Errorf("failed to insert version data, err: %s", err) + return e.ErrCreateDeliveryVersion.AddErr(fmt.Errorf("failed to insert delivery version: %s", versionObj.Version)) + } + + err = buildDeliveryCharts(chartDataMap, versionObj, args.DeliveryVersionChartData, logger) + if err != nil { + return err + } + + return nil +} + +func RetryCreateHelmDeliveryVersion(projectName, versionName string, logger *zap.SugaredLogger) error { + deliveryVersion, err := commonrepo.NewDeliveryVersionColl().Get(&commonrepo.DeliveryVersionArgs{ + ProductName: projectName, + Version: versionName, + }) + if err != nil { + logger.Errorf("failed to query delivery version data, verisonName: %s, error: %s", versionName, err) + return fmt.Errorf("failed to query delivery version data, verisonName: %s", versionName) + } + + if deliveryVersion.Status != setting.DeliveryVersionStatusFailed { + return fmt.Errorf("can't reCreate version with status:%s", deliveryVersion.Status) + } + + argsBytes, err := json.Marshal(deliveryVersion.CreateArgument) + if err != nil { + return errors.Wrapf(err, "failed to marshal arguments, versionName: %s err: %s", deliveryVersion.Version, err) + } + createArgs := new(DeliveryVersionChartData) + err = json.Unmarshal(argsBytes, createArgs) + if err != nil { + return errors.Wrapf(err, "failed to unMarshal arguments, versionName: %s err: %s", deliveryVersion.Version, err) + } + + productInfoSnap := deliveryVersion.ProductEnvInfo + + distributes, err := commonrepo.NewDeliveryDistributeColl().Find(&commonrepo.DeliveryDistributeArgs{ + DistributeType: config.Chart, + ReleaseID: deliveryVersion.ID.Hex(), + }) + if err != nil { + return errors.Wrapf(err, "failed to query distrubutes, versionName: %s", deliveryVersion.Version) + } + + // for charts has been successfully handled, download charts directly + successCharts := sets.NewString() + for _, distribute := range distributes { + if distribute.DistributeType != config.Chart { + continue + } + _, err := downloadChart(deliveryVersion, distribute) + if err != nil { + log.Errorf("failed to download chart from chart repo, chartName: %s, err: %s", distribute.ChartName, err) + continue + } + successCharts.Insert(distribute.ChartName) + } + + chartsToBeHandled := make([]*CreateHelmDeliveryVersionChartData, 0) + for _, chartConfig := range createArgs.ChartDatas { + if successCharts.Has(chartConfig.ServiceName) { + continue + } + chartsToBeHandled = append(chartsToBeHandled, chartConfig) + } + + chartDataMap, err := prepareChartData(chartsToBeHandled, productInfoSnap) + if err != nil { + return e.ErrCreateDeliveryVersion.AddErr(err) + } + + err = buildDeliveryCharts(chartDataMap, deliveryVersion, createArgs, logger) + if err != nil { + return err + } + + // update status + deliveryVersion.Status = setting.DeliveryVersionStatusRetrying + updateVersionStatus(deliveryVersion.Version, deliveryVersion.ProductName, deliveryVersion.Status, deliveryVersion.Error) + + return nil +} + +func ListDeliveryServiceNames(productName string, log *zap.SugaredLogger) ([]string, error) { + serviceNames := sets.String{} + + version := new(commonrepo.DeliveryVersionArgs) + version.ProductName = productName + deliveryVersions, err := FindDeliveryVersion(version, log) + if err != nil { + log.Errorf("FindDeliveryVersion failed, err:%v", err) + return serviceNames.List(), err + } + + for _, deliveryVersion := range deliveryVersions { + deliveryDeployArgs := new(commonrepo.DeliveryDeployArgs) + deliveryDeployArgs.ReleaseID = deliveryVersion.ID.Hex() + deliveryDeploys, err := FindDeliveryDeploy(deliveryDeployArgs, log) + if err != nil { + log.Errorf("FindDeliveryDeploy failed, ReleaseID:%s, err:%v", deliveryVersion.ID, err) + continue + } + for _, deliveryDeploy := range deliveryDeploys { + serviceNames.Insert(deliveryDeploy.ServiceName) + } + } + + return serviceNames.UnsortedList(), nil +} + +func downloadChart(deliveryVersion *commonmodels.DeliveryVersion, chartInfo *commonmodels.DeliveryDistribute) (string, error) { + productName, versionName := deliveryVersion.ProductName, deliveryVersion.Version + chartTGZName := fmt.Sprintf("%s-%s.tgz", chartInfo.ChartName, chartInfo.ChartVersion) + chartTGZFileParent, err := makeChartTGZFileDir(productName, versionName) + if err != nil { + return "", err + } + + chartTGZFilePath := filepath.Join(chartTGZFileParent, chartTGZName) + if _, err := os.Stat(chartTGZFilePath); err == nil { + // local cache exists + return chartTGZFilePath, nil + } + + chartRepo, err := getChartRepoData(chartInfo.ChartRepoName) + if err != nil { + return "", err + } + hClient, err := helmtool.NewClient() + if err != nil { + return "", err + } + + chartRef := fmt.Sprintf("%s/%s", chartRepo.RepoName, chartInfo.ChartName) + return chartTGZFilePath, hClient.DownloadChart(commonservice.GeneHelmRepo(chartRepo), chartRef, chartInfo.ChartVersion, chartTGZFileParent, false) +} + +func getChartDistributeInfo(releaseID, chartName string, log *zap.SugaredLogger) (*commonmodels.DeliveryDistribute, error) { + distributes, _ := FindDeliveryDistribute(&commonrepo.DeliveryDistributeArgs{ + ReleaseID: releaseID, + ChartName: chartName, + DistributeType: config.Chart, + }, log) + + if len(distributes) != 1 { + log.Warnf("find chart %s failed, expect count %d, found count %d, release_id: %s", chartName, 1, len(distributes), releaseID) + return nil, fmt.Errorf("can't find target charts") + } + + chartInfo := distributes[0] + return chartInfo, nil +} + +func DownloadDeliveryChart(projectName, version string, chartName string, log *zap.SugaredLogger) ([]byte, string, error) { + + filePath, err := preDownloadChart(projectName, version, chartName, log) + if err != nil { + return nil, "", err + } + + fileBytes, err := os.ReadFile(filePath) + if err != nil { + return nil, "", err + } + + return fileBytes, filepath.Base(filePath), err +} + +func preDownloadChart(projectName, versionName, chartName string, log *zap.SugaredLogger) (string, error) { + deliveryInfo, err := GetDeliveryVersion(&commonrepo.DeliveryVersionArgs{ + ProductName: projectName, + Version: versionName, + }, log) + if err != nil { + return "", fmt.Errorf("failed to query delivery info") + } + + chartInfo, err := getChartDistributeInfo(deliveryInfo.ID.Hex(), chartName, log) + if err != nil { + return "", err + } + // prepare chart data + filePath, err := downloadChart(deliveryInfo, chartInfo) + if err != nil { + return "", err + } + return filePath, err +} + +func getIndexInfoFromChartRepo(chartRepoName string) (*repo.IndexFile, error) { + chartRepo, err := getChartRepoData(chartRepoName) + if err != nil { + return nil, err + } + hClient, err := helmtool.NewClient() + if err != nil { + return nil, errors.Wrapf(err, "failed to create chart repo client") + } + return hClient.FetchIndexYaml(commonservice.GeneHelmRepo(chartRepo)) +} + +func fillChartUrl(charts []*DeliveryVersionPayloadChart, chartRepoName string) error { + index, err := getIndexInfoFromChartRepo(chartRepoName) + if err != nil { + return err + } + chartMap := make(map[string]*DeliveryVersionPayloadChart) + for _, chart := range charts { + chartMap[chart.ChartName] = chart + } + + for name, entries := range index.Entries { + chart, ok := chartMap[name] + if !ok { + continue + } + if len(entries) == 0 { + continue + } + + for _, entry := range entries { + if entry.Version == chart.ChartVersion { + if len(entry.URLs) > 0 { + chart.ChartUrl = entry.URLs[0] + } + break + } + } + } + return nil +} + +func GetChartVersion(chartName, chartRepoName string) ([]*ChartVersionResp, error) { + + index, err := getIndexInfoFromChartRepo(chartRepoName) + if err != nil { + return nil, err + } + + chartNameList := strings.Split(chartName, ",") + chartNameSet := sets.NewString(chartNameList...) + existedChartSet := sets.NewString() + + ret := make([]*ChartVersionResp, 0) + + for name, entry := range index.Entries { + if !chartNameSet.Has(name) { + continue + } + if len(entry) == 0 { + continue + } + latestEntry := entry[0] + + // generate suggested next chart version + nextVersion := latestEntry.Version + t, err := semver.Make(latestEntry.Version) + if err != nil { + log.Errorf("failed to parse current version: %s, err: %s", latestEntry.Version, err) + } else { + t.Patch = t.Patch + 1 + nextVersion = t.String() + } + + ret = append(ret, &ChartVersionResp{ + ChartName: name, + ChartVersion: latestEntry.Version, + NextChartVersion: nextVersion, + }) + existedChartSet.Insert(name) + } + + for _, singleChartName := range chartNameList { + if existedChartSet.Has(singleChartName) { + continue + } + ret = append(ret, &ChartVersionResp{ + ChartName: singleChartName, + ChartVersion: "", + NextChartVersion: "1.0.0", + }) + } + + return ret, nil +} + +func preDownloadAndUncompressChart(projectName, versionName, chartName string, log *zap.SugaredLogger) (string, error) { + deliveryInfo, err := GetDeliveryVersion(&commonrepo.DeliveryVersionArgs{ + ProductName: projectName, + Version: versionName, + }, log) + if err != nil { + return "", fmt.Errorf("failed to query delivery info") + } + + chartDistribute, err := getChartDistributeInfo(deliveryInfo.ID.Hex(), chartName, log) + if err != nil { + return "", err + } + dstDir := getChartExpandDir(projectName, versionName) + dstDir = filepath.Join(dstDir, fmt.Sprintf("%s-%s", chartDistribute.ChartName, chartDistribute.ChartVersion)) + + filePath, err := preDownloadChart(projectName, versionName, chartName, log) + if err != nil { + return "", err + } + + file, err := os.Open(filePath) + if err != nil { + return "", errors.Wrap(err, "unable to open tarball") + } + defer func() { _ = file.Close() }() + + err = chartutil.Expand(dstDir, file) + if err != nil { + log.Errorf("failed to uncompress file: %s", filePath) + return "", errors.Wrapf(err, "failed to uncompress file") + } + return dstDir, nil +} + +func PreviewDeliveryChart(projectName, version, chartName string, log *zap.SugaredLogger) (*DeliveryChartResp, error) { + dstDir, err := preDownloadAndUncompressChart(projectName, version, chartName, log) + if err != nil { + return nil, err + } + + ret := &DeliveryChartResp{ + FileInfos: make([]*types.FileInfo, 0), + } + + var fis []*types.FileInfo + files, err := os.ReadDir(filepath.Join(dstDir, chartName)) + if err != nil { + return nil, err + } + + for _, file := range files { + info, _ := file.Info() + if info == nil { + continue + } + fi := &types.FileInfo{ + Parent: "", + Name: file.Name(), + Size: info.Size(), + Mode: file.Type(), + ModTime: info.ModTime().Unix(), + IsDir: file.IsDir(), + } + + fis = append(fis, fi) + } + ret.FileInfos = fis + return ret, nil +} + +// load chart file infos +func loadChartFileInfos(fileDir, chartName string, dir string) ([]*types.FileInfo, error) { + var fis []*types.FileInfo + files, err := os.ReadDir(filepath.Join(fileDir, chartName, dir)) + if err != nil { + return nil, e.ErrFilePath.AddDesc(err.Error()) + } + + for _, file := range files { + info, _ := file.Info() + if info == nil { + continue + } + fi := &types.FileInfo{ + Parent: dir, + Name: file.Name(), + Size: info.Size(), + Mode: file.Type(), + ModTime: info.ModTime().Unix(), + IsDir: file.IsDir(), + } + fis = append(fis, fi) + } + return fis, nil +} + +func GetDeliveryChartFilePath(args *DeliveryChartFilePathArgs, log *zap.SugaredLogger) ([]*types.FileInfo, error) { + projectName, version, chartName := args.ProjectName, args.Version, args.ChartName + dstDir, err := preDownloadAndUncompressChart(projectName, version, chartName, log) + if err != nil { + return nil, nil + } + + fileInfos, err := loadChartFileInfos(dstDir, chartName, args.Dir) + if err != nil { + return nil, err + } + return fileInfos, nil +} + +func GetDeliveryChartFileContent(args *DeliveryChartFileContentArgs, log *zap.SugaredLogger) (string, error) { + projectName, version, chartName := args.ProjectName, args.Version, args.ChartName + dstDir, err := preDownloadAndUncompressChart(projectName, version, chartName, log) + if err != nil { + return "", nil + } + + file := filepath.Join(dstDir, chartName, args.FilePath, args.FileName) + fileContent, err := os.ReadFile(file) + if err != nil { + log.Errorf("Failed to read file %s, err: %s", file, err) + return "", e.ErrFileContent.AddDesc(err.Error()) + } + + return string(fileContent), nil +} + +func ApplyDeliveryGlobalVariables(args *DeliveryVariablesApplyArgs, logger *zap.SugaredLogger) (interface{}, error) { + ret := new(DeliveryVariablesApplyArgs) + for _, chartData := range args.ChartDatas { + mergedYaml, err := yamlutil.Merge([][]byte{[]byte(chartData.ValuesYamlContent), []byte(args.GlobalVariables)}) + if err != nil { + logger.Errorf("failed to merge gobal variables for service: %s", chartData.ServiceName) + return nil, errors.Wrapf(err, "failed to merge global variables for service: %s", chartData.ServiceName) + } + ret.ChartDatas = append(ret.ChartDatas, &CreateHelmDeliveryVersionChartData{ + ServiceName: chartData.ServiceName, + ValuesYamlContent: string(mergedYaml), + }) + } + return ret, nil } diff --git a/pkg/microservice/aslan/core/setting/handler/user.go b/pkg/microservice/aslan/core/environment/handler/bundle.go similarity index 76% rename from pkg/microservice/aslan/core/setting/handler/user.go rename to pkg/microservice/aslan/core/environment/handler/bundle.go index c453d691a39fb127f866adb435e752d968e241b0..af39f02c4c3d7b6db0767fde3482b4da46cbceab 100644 --- a/pkg/microservice/aslan/core/setting/handler/user.go +++ b/pkg/microservice/aslan/core/environment/handler/bundle.go @@ -19,13 +19,13 @@ package handler import ( "github.com/gin-gonic/gin" - settingservice "github.com/koderover/zadig/pkg/microservice/aslan/core/setting/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" internalhandler "github.com/koderover/zadig/pkg/shared/handler" ) -func GetUserKubeConfig(c *gin.Context) { +func GetBundleResources(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = settingservice.GetUserKubeConfig(ctx.User.Name, ctx.User.ID, ctx.User.IsSuperUser, ctx.Logger) + ctx.Resp, ctx.Err = service.GetBundleResources(ctx.Logger) } diff --git a/pkg/microservice/aslan/core/environment/handler/common_env_cfg.go b/pkg/microservice/aslan/core/environment/handler/common_env_cfg.go new file mode 100644 index 0000000000000000000000000000000000000000..d178838e81b136495f73f111d680181572feafb4 --- /dev/null +++ b/pkg/microservice/aslan/core/environment/handler/common_env_cfg.go @@ -0,0 +1,158 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "strconv" + + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/log" +) + +func DeleteCommonEnvCfg(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + envName := c.Param("envName") + productName := c.Query("projectName") + commonEnvCfgType := c.Query("commonEnvCfgType") + objectName := c.Param("objectName") + if envName == "" || productName == "" || objectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("param envName or projectName or objectName is invalid") + return + } + internalhandler.InsertOperationLog(c, ctx.UserName, productName, "删除", "环境配置", fmt.Sprintf("%s:%s:%s", envName, commonEnvCfgType, objectName), "", ctx.Logger) + + ctx.Err = service.DeleteCommonEnvCfg(envName, productName, objectName, config.CommonEnvCfgType(commonEnvCfgType), ctx.Logger) +} + +func CreateCommonEnvCfg(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(models.CreateUpdateCommonEnvCfgArgs) + data, err := c.GetRawData() + if err != nil { + log.Errorf("CreateCommonEnvCfg c.GetRawData() err : %v", err) + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("CreateCommonEnvCfg json.Unmarshal err : %v", err) + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } + internalhandler.InsertOperationLog(c, ctx.UserName, c.Query("projectName"), "新建", "环境配置", fmt.Sprintf("%s:%s", args.EnvName, args.CommonEnvCfgType), string(data), ctx.Logger) + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } + if args.YamlData == "" { + ctx.Err = e.ErrInvalidParam + return + } + args.EnvName = c.Param("envName") + args.ProductName = c.Query("projectName") + ctx.Err = service.CreateCommonEnvCfg(args, ctx.UserName, ctx.Logger) +} + +func UpdateCommonEnvCfg(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(models.CreateUpdateCommonEnvCfgArgs) + data, err := c.GetRawData() + if err != nil { + log.Errorf("UpdateCommonEnvCfg c.GetRawData() err : %v", err) + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("UpdateCommonEnvCfg json.Unmarshal err : %v", err) + } + internalhandler.InsertOperationLog(c, ctx.UserName, c.Query("projectName"), "更新", "环境配置", fmt.Sprintf("%s:%s:%s", args.EnvName, args.CommonEnvCfgType, args.Name), string(data), ctx.Logger) + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } + if len(args.YamlData) == 0 { + ctx.Err = e.ErrInvalidParam.AddDesc("yaml info can't be nil") + return + } + args.EnvName = c.Param("envName") + args.ProductName = c.Query("projectName") + isRollBack := false + if len(c.Query("rollback")) > 0 { + isRollBack, err = strconv.ParseBool(c.Query("rollback")) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + } + + ctx.Err = service.UpdateCommonEnvCfg(args, ctx.UserName, isRollBack, ctx.Logger) +} + +func ListCommonEnvCfgHistory(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(service.ListCommonEnvCfgHistoryArgs) + args.EnvName = c.Param("envName") + args.ProjectName = c.Query("projectName") + args.CommonEnvCfgType = config.CommonEnvCfgType(c.Query("commonEnvCfgType")) + args.Name = c.Param("objectName") + + ctx.Resp, ctx.Err = service.ListEnvResourceHistory(args, ctx.Logger) +} + +func ListLatestEnvCfg(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(service.ListCommonEnvCfgHistoryArgs) + if err := c.ShouldBindQuery(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + ctx.Resp, ctx.Err = service.ListLatestEnvResources(args, ctx.Logger) +} + +func SyncEnvResource(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := &service.SyncEnvResourceArg{ + EnvName: c.Param("envName"), + ProductName: c.Query("projectName"), + Name: c.Param("objectName"), + Type: c.Param("type"), + } + ctx.Err = service.SyncEnvResource(args, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/environment/handler/configmap.go b/pkg/microservice/aslan/core/environment/handler/configmap.go index 644e74b5c5d6fda9e743af2014c31bd2e8477f66..1142e614e464fb8eeeccfb1d98d143063eb93ff2 100644 --- a/pkg/microservice/aslan/core/environment/handler/configmap.go +++ b/pkg/microservice/aslan/core/environment/handler/configmap.go @@ -35,48 +35,14 @@ func ListConfigMaps(c *gin.Context) { defer func() { internalhandler.JSONResponse(c, ctx) }() args := &service.ListConfigMapArgs{ - EnvName: c.Query("envName"), - ProductName: c.Query("productName"), + EnvName: c.Param("envName"), + ProductName: c.Query("projectName"), ServiceName: c.Query("serviceName"), } ctx.Resp, ctx.Err = service.ListConfigMaps(args, ctx.Logger) } -func UpdateConfigMap(c *gin.Context) { - ctx := internalhandler.NewContext(c) - defer func() { internalhandler.JSONResponse(c, ctx) }() - - args := new(service.UpdateConfigMapArgs) - data, err := c.GetRawData() - if err != nil { - log.Errorf("UpdateConfigMap c.GetRawData() err : %v", err) - } - if err = json.Unmarshal(data, args); err != nil { - log.Errorf("UpdateConfigMap json.Unmarshal err : %v", err) - } - internalhandler.InsertOperationLog(c, ctx.Username, args.ProductName, "更新", "集成环境-服务-configMap", fmt.Sprintf("环境名称:%s,服务名称:%s", args.EnvName, args.ServiceName), string(data), ctx.Logger) - c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) - - if err := c.BindJSON(args); err != nil { - ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) - return - } - if len(args.Data) == 0 { - ctx.Err = e.ErrInvalidParam - return - } - - // TODO: REVERT AUTH is disabled - //authed := service.CtrlMgr.IsProductAuthed(ctx.UserIDStr, args.ProductOwner, args.ProductName, collection.ProductWritePermission, ctx.Logger) - //if !authed { - // ctx.Err = e.ErrUpdateConfigMap.AddDesc(e.ProductAccessDeniedErrMsg) - // return - //} - - ctx.Err = service.UpdateConfigMap(args.EnvName, args, ctx.User.Name, ctx.User.ID, ctx.Logger) -} - func RollBackConfigMap(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() @@ -89,7 +55,7 @@ func RollBackConfigMap(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("RollBackConfigMap json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, args.ProductName, "回滚", "集成环境-服务-configMap", fmt.Sprintf("环境名称:%s,服务名称:%s", args.EnvName, args.ServiceName), string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, args.ProductName, "回滚", "环境-服务-configMap", fmt.Sprintf("环境名称:%s,服务名称:%s", args.EnvName, args.ServiceName), string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) if err := c.BindJSON(args); err != nil { @@ -102,5 +68,15 @@ func RollBackConfigMap(c *gin.Context) { return } - ctx.Err = service.RollBackConfigMap(args.EnvName, args, ctx.User.Name, ctx.User.ID, ctx.Logger) + ctx.Err = service.RollBackConfigMap(args.EnvName, args, ctx.UserName, ctx.UserID, ctx.Logger) +} + +func MigrateHistoryConfigMaps(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + envName := c.Query("envName") + productName := c.Query("projectName") + + ctx.Resp, ctx.Err = service.MigrateHistoryConfigMaps(envName, productName, ctx.Logger) } diff --git a/pkg/microservice/aslan/core/environment/handler/debug.go b/pkg/microservice/aslan/core/environment/handler/debug.go new file mode 100644 index 0000000000000000000000000000000000000000..72e53caa6c95879245bd3fcba14dad8b32586c90 --- /dev/null +++ b/pkg/microservice/aslan/core/environment/handler/debug.go @@ -0,0 +1,42 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + "github.com/koderover/zadig/pkg/types" +) + +func PatchDebugContainer(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + envName := c.Param("env") + podName := c.Param("podName") + projectName := c.Query("projectName") + debugImage := c.Query("debugImage") + if debugImage == "" { + debugImage = types.DebugImage + } + + internalhandler.InsertOperationLog(c, ctx.UserName, projectName, "启动调试容器", "环境", envName, "", ctx.Logger) + + ctx.Err = service.PatchDebugContainer(c, projectName, envName, podName, debugImage) +} diff --git a/pkg/microservice/aslan/core/environment/handler/environment.go b/pkg/microservice/aslan/core/environment/handler/environment.go index 18a493e9da0438e19423917a0166e7489a46df58..9380c3fad3da82ecf1faab619f835f0e973865d2 100644 --- a/pkg/microservice/aslan/core/environment/handler/environment.go +++ b/pkg/microservice/aslan/core/environment/handler/environment.go @@ -19,12 +19,15 @@ package handler import ( "bytes" "encoding/json" + "errors" "fmt" "io/ioutil" + "net/http" "strconv" "strings" "github.com/gin-gonic/gin" + "k8s.io/apimachinery/pkg/util/sets" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" @@ -38,9 +41,8 @@ import ( "github.com/koderover/zadig/pkg/tool/log" ) -type UpdateEnvs struct { - EnvNames []string `json:"env_names"` - UpdateType string `json:"update_type"` +type DeleteProductServicesRequest struct { + ServiceNames []string `json:"service_names"` } type ChartInfoArgs struct { @@ -52,89 +54,135 @@ type NamespaceResource struct { Ingresses []resource.Ingress `json:"ingresses"` } -// ListProducts list all product information -// Args: projectName, which is formerly known as productName, is the primary key of the project in our system +type UpdateProductRegistryRequest struct { + RegistryID string `json:"registry_id"` +} + func ListProducts(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - - if c.Query("projectName") != "" { - isProduction := c.Query("production") - var envFilter string - switch isProduction { - case "true": - envFilter = setting.ProdENV - case "false": - envFilter = setting.TestENV - default: - envFilter = isProduction - } - ctx.Resp, ctx.Err = service.ListProductsV2(c.Query("projectName"), envFilter, ctx.User.Name, ctx.User.ID, ctx.User.IsSuperUser, ctx.Logger) + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam return } - ctx.Resp, ctx.Err = service.ListProducts(c.Query("productName"), c.Query("envType"), ctx.User.Name, ctx.User.ID, ctx.User.IsSuperUser, ctx.Logger) -} -func AutoCreateProduct(c *gin.Context) { - ctx := internalhandler.NewContext(c) - defer func() { internalhandler.JSONResponse(c, ctx) }() + envNames, found := internalhandler.GetResourcesInHeader(c) + if found && len(envNames) == 0 { + ctx.Resp = []*service.ProductResp{} + return + } - ctx.Resp = service.AutoCreateProduct(c.Param("productName"), c.Query("envType"), ctx.RequestID, ctx.Logger) + ctx.Resp, ctx.Err = service.ListProducts(projectName, envNames, ctx.Logger) } -func AutoUpdateProduct(c *gin.Context) { +func UpdateMultiProducts(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - args := new(UpdateEnvs) + if c.Query("helm") == "true" { + updateMultiHelmEnv(c, ctx) + return + } + args := make([]*service.UpdateEnv, 0) data, err := c.GetRawData() if err != nil { - log.Errorf("AutoUpdateProduct c.GetRawData() err : %v", err) - } - if err = json.Unmarshal(data, args); err != nil { - log.Errorf("AutoUpdateProduct json.Unmarshal err : %v", err) + log.Errorf("UpdateMultiProducts c.GetRawData() err : %v", err) + ctx.Err = e.ErrInvalidParam.AddErr(err) + return } - internalhandler.InsertOperationLog(c, ctx.Username, c.Param("productName"), "自动更新", "集成环境", strings.Join(args.EnvNames, ","), string(data), ctx.Logger) - c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) - - if err := c.BindJSON(args); err != nil { - ctx.Logger.Error(err) - ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + if err = json.Unmarshal(data, &args); err != nil { + log.Errorf("UpdateMultiProducts json.Unmarshal err : %v", err) + ctx.Err = e.ErrInvalidParam.AddErr(err) return } + var envNames []string + for _, arg := range args { + envNames = append(envNames, arg.EnvName) + } + allowedEnvs, found := internalhandler.GetResourcesInHeader(c) + if found { + allowedSet := sets.NewString(allowedEnvs...) + currentSet := sets.NewString(envNames...) + if !allowedSet.IsSuperset(currentSet) { + c.String(http.StatusForbidden, "not all input envs are allowed, allowed envs are %v", allowedEnvs) + return + } + } + + internalhandler.InsertOperationLog(c, ctx.UserName, c.Query("projectName"), "更新", "环境", strings.Join(envNames, ","), string(data), ctx.Logger) force, _ := strconv.ParseBool(c.Query("force")) - ctx.Resp, ctx.Err = service.AutoUpdateProduct(args.EnvNames, c.Param("productName"), ctx.User.ID, ctx.User.IsSuperUser, ctx.RequestID, force, ctx.Logger) + ctx.Resp, ctx.Err = service.AutoUpdateProduct(args, envNames, c.Query("projectName"), ctx.RequestID, force, ctx.Logger) } -func CreateHelmProduct(c *gin.Context) { - ctx := internalhandler.NewContext(c) - defer func() { internalhandler.JSONResponse(c, ctx) }() +func createHelmProduct(c *gin.Context, ctx *internalhandler.Context) { + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("projectName can not be empty") + return + } + createArgs := make([]*service.CreateHelmProductArg, 0) + data, err := c.GetRawData() + if err != nil { + log.Errorf("CreateHelmProduct c.GetRawData() err : %v", err) + } else if err = json.Unmarshal(data, &createArgs); err != nil { + log.Errorf("CreateHelmProduct json.Unmarshal err : %v", err) + } + if err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + + envNameList := make([]string, 0) + for _, arg := range createArgs { + if arg.EnvName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("envName is empty") + return + } + arg.ProductName = projectName + envNameList = append(envNameList, arg.EnvName) + } + internalhandler.InsertOperationLog(c, ctx.UserName, projectName, "新增", "环境", strings.Join(envNameList, "-"), string(data), ctx.Logger) - if c.Param("productName") == "" { - ctx.Err = e.ErrInvalidParam.AddDesc("productName can not be null!") + ctx.Err = service.CreateHelmProduct( + projectName, ctx.UserName, ctx.RequestID, createArgs, ctx.Logger, + ) +} + +func copyHelmProduct(c *gin.Context, ctx *internalhandler.Context) { + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("projectName can not be empty") return } - args := new(service.CreateHelmProductArg) + createArgs := make([]*service.CreateHelmProductArg, 0) data, err := c.GetRawData() if err != nil { - log.Errorf("CreateProduct c.GetRawData() err : %v", err) + log.Errorf("copyHelmProduct c.GetRawData() err : %s", err) + } else if err = json.Unmarshal(data, &createArgs); err != nil { + log.Errorf("copyHelmProduct json.Unmarshal err : %s", err) } - if err = json.Unmarshal(data, args); err != nil { - log.Errorf("CreateProduct json.Unmarshal err : %v", err) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return } - args.ProductName = c.Param("productName") - if args.EnvName == "" { - ctx.Err = e.ErrInvalidParam.AddDesc("envName can not be null!") - return + envNameCopyList := make([]string, 0) + for _, arg := range createArgs { + if arg.EnvName == "" || arg.BaseEnvName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("envName or baseEnvName is empty") + return + } + arg.ProductName = projectName + envNameCopyList = append(envNameCopyList, arg.BaseEnvName+"-->"+arg.EnvName) } - internalhandler.InsertOperationLog(c, ctx.Username, args.ProductName, "新增", "集成环境", args.EnvName, string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, projectName, "复制", "环境", strings.Join(envNameCopyList, ","), string(data), ctx.Logger) - ctx.Err = service.CreateHelmProduct( - ctx.Username, ctx.RequestID, args, ctx.Logger, + ctx.Err = service.CopyHelmProduct( + projectName, ctx.UserName, ctx.RequestID, createArgs, ctx.Logger, ) } @@ -142,6 +190,19 @@ func CreateProduct(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() + if c.Query("auto") == "true" { + ctx.Resp = service.AutoCreateProduct(c.Query("projectName"), c.Query("envType"), ctx.RequestID, ctx.Logger) + return + } + if c.Query("helm") == "true" { + if c.Query("scene") == "copy" { + copyHelmProduct(c, ctx) + return + } + createHelmProduct(c, ctx) + return + } + args := new(commonmodels.Product) data, err := c.GetRawData() if err != nil { @@ -150,7 +211,17 @@ func CreateProduct(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("CreateProduct json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, args.ProductName, "新增", "集成环境", args.EnvName, string(data), ctx.Logger) + + allowedClusters, found := internalhandler.GetResourcesInHeader(c) + if found { + allowedSet := sets.NewString(allowedClusters...) + if !allowedSet.Has(args.ClusterID) { + c.String(http.StatusForbidden, "permission denied for cluster %s", args.ClusterID) + return + } + } + + internalhandler.InsertOperationLog(c, ctx.UserName, args.ProductName, "新增", "环境", args.EnvName, string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) if err := c.BindJSON(args); err != nil { @@ -163,28 +234,35 @@ func CreateProduct(c *gin.Context) { return } - args.UpdateBy = ctx.Username + args.UpdateBy = ctx.UserName ctx.Err = service.CreateProduct( - ctx.Username, ctx.RequestID, args, ctx.Logger, + ctx.UserName, ctx.RequestID, args, ctx.Logger, ) } +type UpdateProductParams struct { + ServiceNames []string `json:"service_names"` + commonmodels.Product +} + func UpdateProduct(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - envName := c.Query("envName") - productName := c.Param("productName") + envName := c.Param("name") + projectName := c.Query("projectName") - args := new(commonmodels.Product) + args := new(UpdateProductParams) data, err := c.GetRawData() if err != nil { log.Errorf("UpdateProduct c.GetRawData() err : %v", err) } if err = json.Unmarshal(data, args); err != nil { log.Errorf("UpdateProduct json.Unmarshal err : %v", err) + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return } - internalhandler.InsertOperationLog(c, ctx.Username, productName, "更新", "集成环境", envName, string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, projectName, "更新", "环境变量", envName, string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) if err := c.BindJSON(args); err != nil { @@ -193,10 +271,46 @@ func UpdateProduct(c *gin.Context) { } force, _ := strconv.ParseBool(c.Query("force")) + serviceNames := sets.String{} + for _, kv := range args.Vars { + for _, s := range kv.Services { + serviceNames.Insert(s) + } + } // update product asynchronously - ctx.Err = service.UpdateProductV2(envName, productName, ctx.Username, ctx.RequestID, force, args.Vars, ctx.Logger) + ctx.Err = service.UpdateProductV2(envName, projectName, ctx.UserName, ctx.RequestID, serviceNames.List(), force, args.Vars, ctx.Logger) if ctx.Err != nil { - ctx.Logger.Errorf("failed to update product %s %s: %v", envName, productName, ctx.Err) + ctx.Logger.Errorf("failed to update product %s %s: %v", envName, projectName, ctx.Err) + } +} + +func UpdateProductRegistry(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + envName := c.Param("name") + projectName := c.Query("projectName") + args := new(UpdateProductRegistryRequest) + data, err := c.GetRawData() + if err != nil { + log.Errorf("UpdateProduct c.GetRawData() err : %v", err) + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("UpdateProduct json.Unmarshal err : %v", err) + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } + internalhandler.InsertOperationLog(c, ctx.UserName, projectName, "更新", "环境", envName, string(data), ctx.Logger) + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } + ctx.Err = service.UpdateProductRegistry(envName, projectName, args.RegistryID, ctx.Logger) + if ctx.Err != nil { + ctx.Logger.Errorf("failed to update product %s %s: %v", envName, args.RegistryID, ctx.Err) } } @@ -204,11 +318,11 @@ func UpdateProductRecycleDay(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - envName := c.Query("envName") - productName := c.Param("productName") + envName := c.Param("name") + projectName := c.Query("projectName") recycleDayStr := c.Query("recycleDay") - internalhandler.InsertOperationLog(c, ctx.Username, productName, "更新", "集成环境-环境回收", envName, "", ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, projectName, "更新", "环境-环境回收", envName, "", ctx.Logger) var ( recycleDay int @@ -224,186 +338,230 @@ func UpdateProductRecycleDay(c *gin.Context) { return } - ctx.Err = service.UpdateProductRecycleDay(envName, productName, recycleDay) + ctx.Err = service.UpdateProductRecycleDay(envName, projectName, recycleDay) } -func UpdateHelmProduct(c *gin.Context) { +func EstimatedValues(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - envName := c.Query("envName") - updateType := c.Query("updateType") - productName := c.Param("productName") + envName := c.Param("name") + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("projectName can't be empty!") + return + } - ctx.Err = service.UpdateHelmProduct(productName, envName, updateType, ctx.Username, ctx.RequestID, nil, ctx.Logger) - if ctx.Err != nil { - ctx.Logger.Errorf("failed to update product %s %s: %v", envName, productName, ctx.Err) + serviceName := c.Query("serviceName") + if serviceName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("serviceName can't be empty!") + return } + + arg := new(service.EstimateValuesArg) + if err := c.ShouldBind(arg); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } + + ctx.Resp, ctx.Err = service.GeneEstimatedValues(projectName, envName, serviceName, c.Query("scene"), c.Query("format"), arg, ctx.Logger) } -func UpdateHelmProductRenderCharts(c *gin.Context) { +func SyncHelmProductRenderset(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - productName := c.Param("productName") - if productName == "" { - ctx.Err = e.ErrInvalidParam.AddDesc("productName can't be empty!") + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("projectName can't be empty!") return } - - envName := c.Query("envName") + envName := c.Param("name") if envName == "" { ctx.Err = e.ErrInvalidParam.AddDesc("envName can't be empty!") return } - arg := new(service.EnvRenderChartArg) - data, err := c.GetRawData() - if err != nil { - log.Errorf("UpdateHelmProductVariable c.GetRawData() err : %v", err) - } - if err = json.Unmarshal(data, arg); err != nil { - log.Errorf("UpdateHelmProductVariable json.Unmarshal err : %v", err) + ctx.Err = service.SyncHelmProductEnvironment(projectName, envName, ctx.RequestID, ctx.Logger) +} + +func generalRequestValidate(c *gin.Context) (string, string, error) { + projectName := c.Query("projectName") + if projectName == "" { + return "", "", errors.New("projectName can't be empty") } - internalhandler.InsertOperationLog(c, ctx.Username, c.Param("productName"), "更新", "更新环境变量", "", string(data), ctx.Logger) - ctx.Err = service.UpdateHelmProductRenderCharts(productName, envName, ctx.Username, ctx.RequestID, arg.ChartValues, ctx.Logger) - if ctx.Err != nil { - ctx.Logger.Errorf("failed to update product Variable %s %s: %v", envName, productName, ctx.Err) + envName := c.Param("name") + if envName == "" { + return "", "", errors.New("envName can't be empty") } + return projectName, envName, nil } -func UpdateHelmProductVariable(c *gin.Context) { +func UpdateHelmProductRenderset(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - envName := c.Query("envName") - productName := c.Param("productName") + projectName, envName, err := generalRequestValidate(c) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } - args := new(ChartInfoArgs) + arg := new(service.EnvRendersetArg) data, err := c.GetRawData() if err != nil { - log.Errorf("UpdateHelmProductVariable c.GetRawData() err : %v", err) + log.Errorf("UpdateHelmProductRenderset c.GetRawData() err : %v", err) } - if err = json.Unmarshal(data, args); err != nil { - log.Errorf("UpdateHelmProductVariable json.Unmarshal err : %v", err) + if err = json.Unmarshal(data, arg); err != nil { + log.Errorf("UpdateHelmProductRenderset json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, c.Param("productName"), "更新", "helm集成环境变量", "", string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, projectName, "更新", "更新环境变量", "", string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) - if err := c.BindJSON(args); err != nil { - ctx.Logger.Error(err) - ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + err = c.BindJSON(arg) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) return } - ctx.Err = service.UpdateHelmProductVariable(productName, envName, ctx.Username, ctx.RequestID, args.ChartInfos, args.ChartInfos, ctx.Logger) + ctx.Err = service.UpdateHelmProductRenderset(projectName, envName, ctx.UserName, ctx.RequestID, arg, ctx.Logger) if ctx.Err != nil { - ctx.Logger.Errorf("failed to update product Variable %s %s: %v", envName, productName, ctx.Err) + ctx.Logger.Errorf("failed to update product Variable %s %s: %v", envName, projectName, ctx.Err) } } -func UpdateMultiHelmEnv(c *gin.Context) { +func UpdateHelmProductDefaultValues(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - if c.Param("productName") == "" { - ctx.Err = e.ErrInvalidParam.AddDesc("productName can not be null!") + projectName, envName, err := generalRequestValidate(c) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) return } - args := new(service.UpdateMultiHelmProductArg) + arg := new(service.EnvRendersetArg) data, err := c.GetRawData() if err != nil { - log.Errorf("CreateProduct c.GetRawData() err : %v", err) + log.Errorf("UpdateHelmProductDefaultValues c.GetRawData() err : %v", err) } - if err = json.Unmarshal(data, args); err != nil { - log.Errorf("CreateProduct json.Unmarshal err : %v", err) + if err = json.Unmarshal(data, arg); err != nil { + log.Errorf("UpdateHelmProductDefaultValues json.Unmarshal err : %v", err) } - args.ProductName = c.Param("productName") + internalhandler.InsertOperationLog(c, ctx.UserName, projectName, "更新", "更新全局变量", envName, string(data), ctx.Logger) + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) - internalhandler.InsertOperationLog(c, ctx.Username, args.ProductName, "更新", "集成环境", strings.Join(args.EnvNames, ","), string(data), ctx.Logger) + err = c.BindJSON(arg) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } - ctx.Resp, ctx.Err = service.UpdateMultipleHelmEnv( - ctx.Username, ctx.RequestID, ctx.User.ID, ctx.User.IsSuperUser, args, ctx.Logger, - ) + ctx.Err = service.UpdateHelmProductDefaultValues(projectName, envName, ctx.UserName, ctx.RequestID, arg, ctx.Logger) } -func UpdateMultiHelmProduct(c *gin.Context) { +func UpdateHelmProductCharts(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - args := new(UpdateEnvs) + projectName, envName, err := generalRequestValidate(c) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + + arg := new(service.EnvRendersetArg) + data, err := c.GetRawData() if err != nil { - log.Errorf("UpdateMultiHelmProduct c.GetRawData() err : %v", err) + log.Errorf("UpdateHelmProductCharts c.GetRawData() err : %v", err) } - if err = json.Unmarshal(data, args); err != nil { - log.Errorf("UpdateMultiHelmProduct json.Unmarshal err : %v", err) + if err = json.Unmarshal(data, arg); err != nil { + log.Errorf("UpdateHelmProductCharts json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, c.Param("productName"), "更新helm环境", "集成环境", strings.Join(args.EnvNames, ","), string(data), ctx.Logger) + + serviceName := make([]string, 0) + for _, cd := range arg.ChartValues { + serviceName = append(serviceName, cd.ServiceName) + } + + internalhandler.InsertOperationLog(c, ctx.UserName, projectName, "更新", "更新服务", fmt.Sprintf("%s:[%s]", envName, strings.Join(serviceName, ",")), string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) - if err := c.BindJSON(args); err != nil { - ctx.Logger.Error(err) - ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + err = c.BindJSON(arg) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) return } - ctx.Resp = service.UpdateMultiHelmProduct(args.EnvNames, args.UpdateType, c.Param("productName"), ctx.User.ID, ctx.User.IsSuperUser, ctx.RequestID, ctx.Logger) + ctx.Err = service.UpdateHelmProductCharts(projectName, envName, ctx.UserName, ctx.RequestID, arg, ctx.Logger) } -func GetProduct(c *gin.Context) { - ctx := internalhandler.NewContext(c) - defer func() { internalhandler.JSONResponse(c, ctx) }() - - envName := c.Query("envName") - productName := c.Param("productName") +func updateMultiHelmEnv(c *gin.Context, ctx *internalhandler.Context) { + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("projectName can not be empty") + return + } - ctx.Resp, ctx.Err = service.GetProduct(ctx.Username, envName, productName, ctx.Logger) -} + args := new(service.UpdateMultiHelmProductArg) + data, err := c.GetRawData() + if err != nil { + log.Errorf("CreateProduct c.GetRawData() err : %v", err) + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("CreateProduct json.Unmarshal err : %v", err) + } + args.ProductName = projectName -func GetProductInfo(c *gin.Context) { - ctx := internalhandler.NewContext(c) - defer func() { internalhandler.JSONResponse(c, ctx) }() + allowedEnvs, found := internalhandler.GetResourcesInHeader(c) + if found { + allowedSet := sets.NewString(allowedEnvs...) + currentSet := sets.NewString(args.EnvNames...) + if !allowedSet.IsSuperset(currentSet) { + c.String(http.StatusForbidden, "not all input envs are allowed, allowed envs are %v", allowedEnvs) + return + } + } - envName := c.Query("envName") - productName := c.Param("productName") + internalhandler.InsertOperationLog(c, ctx.UserName, projectName, "更新", "环境", strings.Join(args.EnvNames, ","), string(data), ctx.Logger) - ctx.Resp, ctx.Err = service.GetProductInfo(ctx.Username, envName, productName, ctx.Logger) + ctx.Resp, ctx.Err = service.UpdateMultipleHelmEnv( + ctx.RequestID, ctx.UserName, args, ctx.Logger, + ) } -func GetProductIngress(c *gin.Context) { +func GetProduct(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - productName := c.Param("productName") - ctx.Resp, ctx.Err = service.GetProductIngress(ctx.Username, productName, ctx.Logger) + projectName := c.Query("projectName") + name := c.Param("name") + + ctx.Resp, ctx.Err = service.GetProduct(ctx.UserName, name, projectName, ctx.Logger) } -func ListRenderCharts(c *gin.Context) { +func GetProductInfo(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - envName := c.Query("envName") - productName := c.Param("productName") + envName := c.Param("name") + projectName := c.Query("projectName") - ctx.Resp, ctx.Err = service.ListRenderCharts(productName, envName, ctx.Logger) - if ctx.Err != nil { - ctx.Logger.Errorf("failed to listRenderCharts %s %s: %v", envName, productName, ctx.Err) - } + ctx.Resp, ctx.Err = service.GetProductInfo(ctx.UserName, envName, projectName, ctx.Logger) } func GetHelmChartVersions(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - envName := c.Query("envName") - productName := c.Param("productName") + envName := c.Param("name") + projectName := c.Query("projectName") - ctx.Resp, ctx.Err = service.GetHelmChartVersions(productName, envName, ctx.Logger) + ctx.Resp, ctx.Err = service.GetHelmChartVersions(projectName, envName, ctx.Logger) if ctx.Err != nil { - ctx.Logger.Errorf("failed to get helmVersions %s %s: %v", envName, productName, ctx.Err) + ctx.Logger.Errorf("failed to get helmVersions %s %s: %v", envName, projectName, ctx.Err) } } @@ -411,71 +569,85 @@ func GetEstimatedRenderCharts(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - productName := c.Query("productName") - if productName == "" { - ctx.Err = e.ErrInvalidParam.AddDesc("productName can't be empty!") + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("projectName can't be empty!") + return } - envName := c.Query("envName") + envName := c.Param("name") if envName == "" { ctx.Err = e.ErrInvalidParam.AddDesc("envName can't be empty!") + return } - ctx.Resp, ctx.Err = service.GetEstimatedRenderCharts(productName, envName, c.Query("serviceName"), ctx.Logger) + ctx.Resp, ctx.Err = service.GetEstimatedRenderCharts(projectName, envName, c.Query("serviceName"), ctx.Logger) if ctx.Err != nil { - ctx.Logger.Errorf("failed to get estimatedRenderCharts %s %s: %v", envName, productName, ctx.Err) + ctx.Logger.Errorf("failed to get estimatedRenderCharts %s %s: %v", envName, projectName, ctx.Err) } } func DeleteProduct(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - envName := c.Query("envName") - internalhandler.InsertOperationLog(c, ctx.Username, c.Param("productName"), "删除", "集成环境", envName, "", ctx.Logger) - ctx.Err = commonservice.DeleteProduct(ctx.Username, envName, c.Param("productName"), ctx.RequestID, ctx.Logger) + envName := c.Param("name") + projectName := c.Query("projectName") + is_delete, err := strconv.ParseBool(c.Query("is_delete")) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalidParam is_delete") + return + } + internalhandler.InsertOperationLog(c, ctx.UserName, projectName, "删除", "环境", envName, "", ctx.Logger) + ctx.Err = service.DeleteProduct(ctx.UserName, envName, projectName, ctx.RequestID, is_delete, ctx.Logger) } -func EnvShare(c *gin.Context) { +func DeleteProductServices(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - productName := c.Param("productName") - args := new(service.ProductParams) + args := new(DeleteProductServicesRequest) data, err := c.GetRawData() if err != nil { - log.Errorf("CreateProduct c.GetRawData() err : %v", err) + log.Errorf("DeleteProductServices c.GetRawData() err : %v", err) + ctx.Err = e.ErrInvalidParam.AddErr(err) + return } if err = json.Unmarshal(data, args); err != nil { - log.Errorf("CreateProduct json.Unmarshal err : %v", err) - } - internalhandler.InsertOperationLog(c, ctx.Username, productName, "更新", "集成环境-环境授权", args.EnvName, string(data), ctx.Logger) - c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) - - if productName == "" { - ctx.Err = e.ErrInvalidParam.AddDesc("productName can't be empty!") + log.Errorf("DeleteProductServices json.Unmarshal err : %v", err) + ctx.Err = e.ErrInvalidParam.AddErr(err) return } + projectName := c.Query("projectName") + envName := c.Param("name") - if err := c.BindJSON(args); err != nil { - ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + // For environment sharing, if the environment is the base environment and the service to be deleted has been deployed in the subenvironment, + // we should prompt the user that `Delete the service in the subenvironment before deleting the service in the base environment`. + svcsInSubEnvs, err := service.CheckServicesDeployedInSubEnvs(c, projectName, envName, args.ServiceNames) + if err != nil { + ctx.Err = err return } + if len(svcsInSubEnvs) > 0 { + data := make(map[string]interface{}, len(svcsInSubEnvs)) + for k, v := range svcsInSubEnvs { + data[k] = v + } - if args.EnvName == "" { - ctx.Err = e.ErrInvalidParam.AddDesc("envName can't be empty!") + ctx.Err = e.NewWithExtras(e.ErrDeleteSvcHasSvcsInSubEnv, "", data) return } - ctx.Err = service.UpdateProductPublic(productName, args, ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, projectName, "删除", "环境的服务", fmt.Sprintf("%s:[%s]", envName, strings.Join(args.ServiceNames, ",")), "", ctx.Logger) + ctx.Err = service.DeleteProductServices(ctx.UserName, ctx.RequestID, envName, projectName, args.ServiceNames, ctx.Logger) } func ListGroups(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - productName := c.Param("productName") - envName := c.Query("envName") + envName := c.Param("name") + projectName := c.Query("projectName") serviceName := c.Query("serviceName") perPageStr := c.Query("perPage") pageStr := c.Query("page") @@ -505,7 +677,7 @@ func ListGroups(c *gin.Context) { } } - ctx.Resp, count, ctx.Err = service.ListGroups(serviceName, envName, productName, perPage, page, ctx.Logger) + ctx.Resp, count, ctx.Err = service.ListGroups(serviceName, envName, projectName, perPage, page, ctx.Logger) c.Writer.Header().Set("X-Total", strconv.Itoa(count)) } @@ -532,6 +704,20 @@ func ListWorkloads(c *gin.Context) { for _, workload := range workloadStat.Workloads { workloadM[workload.Name] = workload } + + // add services in external env data + servicesInExternalEnv, _ := mongodb.NewServicesInExternalEnvColl().List(&mongodb.ServicesInExternalEnvArgs{ + Namespace: args.Namespace, + ClusterID: args.ClusterID, + }) + + for _, serviceInExternalEnv := range servicesInExternalEnv { + workloadM[serviceInExternalEnv.ServiceName] = commonmodels.Workload{ + EnvName: serviceInExternalEnv.EnvName, + ProductName: serviceInExternalEnv.ProductName, + } + } + for index, currentWorkload := range workloads { if existWorkload, ok := workloadM[currentWorkload.Name]; ok { workloads[index].EnvName = existWorkload.EnvName @@ -557,25 +743,26 @@ func ListWorkloads(c *gin.Context) { c.Writer.Header().Set("X-Total", strconv.Itoa(count)) } -// TODO: envName must be a param, while productName can be a query type workloadQueryArgs struct { - Env string `json:"env" form:"env"` - PerPage int `json:"perPage" form:"perPage,default=10"` - Page int `json:"page" form:"page,default=1"` - Filter string `json:"filter" form:"filter"` + ProjectName string `json:"projectName" form:"projectName"` + PerPage int `json:"perPage" form:"perPage,default=10"` + Page int `json:"page" form:"page,default=1"` + Filter string `json:"filter" form:"filter"` } func ListWorkloadsInEnv(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() + envName := c.Param("name") + args := &workloadQueryArgs{} if err := c.ShouldBindQuery(args); err != nil { ctx.Err = err return } - count, services, err := commonservice.ListWorkloadsInEnv(args.Env, c.Param("productName"), args.Filter, args.PerPage, args.Page, ctx.Logger) + count, services, err := commonservice.ListWorkloadsInEnv(envName, args.ProjectName, args.Filter, args.PerPage, args.Page, ctx.Logger) ctx.Resp = &NamespaceResource{ Services: services, } diff --git a/pkg/microservice/aslan/core/environment/handler/export.go b/pkg/microservice/aslan/core/environment/handler/export.go index 6f0a9917c552a901f0950c1700613a83a4d7900b..7419254fc49bb34bbea37652e65c7f56514d710f 100644 --- a/pkg/microservice/aslan/core/environment/handler/export.go +++ b/pkg/microservice/aslan/core/environment/handler/export.go @@ -29,7 +29,7 @@ func ExportYaml(c *gin.Context) { serviceName := c.Query("serviceName") envName := c.Query("envName") - productName := c.Query("productName") + productName := c.Query("projectName") ctx.Resp = service.ExportYaml(envName, productName, serviceName, ctx.Logger) } diff --git a/pkg/microservice/aslan/core/environment/handler/helm.go b/pkg/microservice/aslan/core/environment/handler/helm.go new file mode 100644 index 0000000000000000000000000000000000000000..5280d01f33218a66fd257966180fe26a040e9d57 --- /dev/null +++ b/pkg/microservice/aslan/core/environment/handler/helm.go @@ -0,0 +1,65 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" +) + +func ListReleases(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + envName := c.Param("name") + + args := &service.HelmReleaseQueryArgs{} + if err := c.ShouldBindQuery(args); err != nil { + ctx.Err = err + return + } + + ctx.Resp, ctx.Err = service.ListReleases(args, envName, ctx.Logger) +} + +func GetChartValues(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + envName := c.Param("name") + projectName := c.Query("projectName") + serviceName := c.Query("serviceName") + + ctx.Resp, ctx.Err = service.GetChartValues(projectName, envName, serviceName) +} + +func GetChartInfos(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + envName := c.Param("name") + servicesName := c.Query("serviceName") + projectName := c.Query("projectName") + ctx.Resp, ctx.Err = service.GetChartInfos(projectName, envName, servicesName, ctx.Logger) +} + +func GetImageInfos(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + envName := c.Param("name") + projectName := c.Query("projectName") + servicesName := c.Query("serviceName") + ctx.Resp, ctx.Err = service.GetImageInfos(projectName, envName, servicesName, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/environment/handler/ide.go b/pkg/microservice/aslan/core/environment/handler/ide.go new file mode 100644 index 0000000000000000000000000000000000000000..c700e7cd3a01a5186570bf5a459121bf4a76ffec --- /dev/null +++ b/pkg/microservice/aslan/core/environment/handler/ide.go @@ -0,0 +1,55 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "errors" + + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + "github.com/koderover/zadig/pkg/types" +) + +func PatchWorkload(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + envName := c.Param("name") + serviceName := c.Param("serviceName") + projectName := c.Query("projectName") + + var staretInfo types.StartDevmodeInfo + if err := c.BindJSON(&staretInfo); err != nil { + ctx.Err = errors.New("invalid request body") + return + } + + ctx.Resp, ctx.Err = service.PatchWorkload(c, projectName, envName, serviceName, staretInfo.DevImage) +} + +func RecoverWorkload(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + envName := c.Param("name") + serviceName := c.Param("serviceName") + projectName := c.Query("projectName") + + ctx.Err = service.RecoverWorkload(c, projectName, envName, serviceName) +} diff --git a/pkg/microservice/aslan/core/environment/handler/image.go b/pkg/microservice/aslan/core/environment/handler/image.go index 48bbcd6c6c5f3660e61c79f6b1c0d6fc36ce8dc0..2ee65c80b4f17dad0276080064562a34b92feefd 100644 --- a/pkg/microservice/aslan/core/environment/handler/image.go +++ b/pkg/microservice/aslan/core/environment/handler/image.go @@ -49,8 +49,8 @@ func UpdateStatefulSetContainerImage(c *gin.Context) { } internalhandler.InsertOperationLog( - c, ctx.Username, args.ProductName, - "更新", "集成环境-服务镜像", + c, ctx.UserName, args.ProductName, + "更新", "环境-服务镜像", fmt.Sprintf("环境名称:%s,服务名称:%s,StatefulSet:%s", args.EnvName, args.ServiceName, args.Name), string(data), ctx.Logger) @@ -80,8 +80,8 @@ func UpdateDeploymentContainerImage(c *gin.Context) { } internalhandler.InsertOperationLog( - c, ctx.Username, args.ProductName, - "更新", "集成环境-服务镜像", + c, ctx.UserName, args.ProductName, + "更新", "环境-服务镜像", fmt.Sprintf("环境名称:%s,服务名称:%s,Deployment:%s", args.EnvName, args.ServiceName, args.Name), string(data), ctx.Logger) diff --git a/pkg/microservice/aslan/core/environment/handler/ingress.go b/pkg/microservice/aslan/core/environment/handler/ingress.go new file mode 100644 index 0000000000000000000000000000000000000000..3c16cc6186f20b24453fdd5839371eb0dc4348ea --- /dev/null +++ b/pkg/microservice/aslan/core/environment/handler/ingress.go @@ -0,0 +1,38 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +func ListIngresses(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + envName := c.Param("envName") + productName := c.Query("projectName") + if envName == "" || productName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("param envName or projectName is invalid") + return + } + + ctx.Resp, ctx.Err = service.ListIngresses(envName, productName, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/environment/handler/kube.go b/pkg/microservice/aslan/core/environment/handler/kube.go index c35ea7bbfda5b88d4f52f83cca0a564c1471c86a..07905fa1a8da7d68d9b7c8ff1751731c119b7378 100644 --- a/pkg/microservice/aslan/core/environment/handler/kube.go +++ b/pkg/microservice/aslan/core/environment/handler/kube.go @@ -38,7 +38,7 @@ func ListKubeEvents(c *gin.Context) { defer func() { internalhandler.JSONResponse(c, ctx) }() envName := c.Query("envName") - productName := c.Query("productName") + productName := c.Query("projectName") name := c.Query("name") rtype := c.Query("type") @@ -54,7 +54,9 @@ func ListAvailableNamespaces(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = service.ListAvailableNamespaces(c.Query("clusterId"), ctx.Logger) + clusterID := c.Query("clusterId") + listType := c.Query("type") + ctx.Resp, ctx.Err = service.ListAvailableNamespaces(clusterID, listType, ctx.Logger) } func ListServicePods(c *gin.Context) { @@ -81,9 +83,9 @@ func DeletePod(c *gin.Context) { podName := c.Param("podName") envName := c.Query("envName") - productName := c.Query("productName") + productName := c.Query("projectName") - internalhandler.InsertOperationLog(c, ctx.Username, c.Query("productName"), "重启", "集成环境-服务实例", fmt.Sprintf("环境名称:%s,pod名称:%s", c.Query("envName"), c.Param("podName")), "", ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, c.Query("projectName"), "重启", "环境-服务实例", fmt.Sprintf("环境名称:%s,pod名称:%s", c.Query("envName"), c.Param("podName")), "", ctx.Logger) ctx.Err = service.DeletePod(envName, productName, podName, ctx.Logger) } @@ -92,8 +94,15 @@ func ListPodEvents(c *gin.Context) { defer func() { internalhandler.JSONResponse(c, ctx) }() envName := c.Query("envName") - productName := c.Query("productName") + productName := c.Query("projectName") podName := c.Param("podName") ctx.Resp, ctx.Err = service.ListPodEvents(envName, productName, podName, ctx.Logger) } + +func ListNodes(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.ListAvailableNodes(c.Query("clusterId"), ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/environment/handler/pm_exec.go b/pkg/microservice/aslan/core/environment/handler/pm_exec.go new file mode 100644 index 0000000000000000000000000000000000000000..c10edaa107887e072249e792bf4d11c5d1c28652 --- /dev/null +++ b/pkg/microservice/aslan/core/environment/handler/pm_exec.go @@ -0,0 +1,51 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "strconv" + + "github.com/gin-gonic/gin" + "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +func ConnectSshPmExec(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + projectName := c.Query("projectName") + ip := c.Query("ip") + hostId := c.Query("hostId") + name := c.Param("name") + if projectName == "" || ip == "" || name == "" || hostId == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("param projectName or ip or name or hostId is empty") + } + colsStr := c.DefaultQuery("cols", "135") + cols, err := strconv.Atoi(colsStr) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + } + rowsStr := c.DefaultQuery("rows", "40") + rows, err := strconv.Atoi(rowsStr) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + } + + ctx.Err = service.ConnectSshPmExec(c, ctx.UserName, name, projectName, ip, hostId, cols, rows, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/environment/handler/product.go b/pkg/microservice/aslan/core/environment/handler/product.go index 7ba604f02fd45907bee7e01a7cab2728e3cbdb6f..e8b187d0d1bc74de7d2ccec4963f4efe33d57c8f 100644 --- a/pkg/microservice/aslan/core/environment/handler/product.go +++ b/pkg/microservice/aslan/core/environment/handler/product.go @@ -17,10 +17,14 @@ limitations under the License. package handler import ( + "fmt" + "strconv" + "github.com/gin-gonic/gin" "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" internalhandler "github.com/koderover/zadig/pkg/shared/handler" + "github.com/koderover/zadig/pkg/types" ) func CleanProductCronJob(c *gin.Context) { @@ -35,5 +39,24 @@ func GetInitProduct(c *gin.Context) { defer func() { internalhandler.JSONResponse(c, ctx) }() productTemplateName := c.Param("name") - ctx.Resp, ctx.Err = service.GetInitProduct(productTemplateName, ctx.Logger) + + envType := types.EnvType(c.Query("envType")) + isBaseEnvStr := c.Query("isBaseEnv") + baseEnvName := c.Query("baseEnv") + + if envType == "" { + envType = types.GeneralEnv + } + + var isBaseEnv bool + var err error + if envType == types.ShareEnv { + isBaseEnv, err = strconv.ParseBool(isBaseEnvStr) + if err != nil { + ctx.Err = fmt.Errorf("failed to parse %s to bool: %s", isBaseEnvStr, err) + return + } + } + + ctx.Resp, ctx.Err = service.GetInitProduct(productTemplateName, envType, isBaseEnv, baseEnvName, ctx.Logger) } diff --git a/pkg/microservice/aslan/core/environment/handler/pvc.go b/pkg/microservice/aslan/core/environment/handler/pvc.go new file mode 100644 index 0000000000000000000000000000000000000000..a3417f7bdca2c4baeede1463c671a3a10a782156 --- /dev/null +++ b/pkg/microservice/aslan/core/environment/handler/pvc.go @@ -0,0 +1,38 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +func ListPvcs(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + envName := c.Param("envName") + productName := c.Query("projectName") + if envName == "" || productName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("param envName or projectName is invalid") + return + } + + ctx.Resp, ctx.Err = service.ListPvcs(envName, productName, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/environment/handler/renderset.go b/pkg/microservice/aslan/core/environment/handler/renderset.go index 7ede1b4429a4894871a351092b0b29a6b5dc429b..d3ccd77705fe5029335197e7117e27463b4f325d 100644 --- a/pkg/microservice/aslan/core/environment/handler/renderset.go +++ b/pkg/microservice/aslan/core/environment/handler/renderset.go @@ -17,7 +17,7 @@ limitations under the License. package handler import ( - "encoding/json" + "strings" "github.com/gin-gonic/gin" @@ -25,14 +25,13 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" - "github.com/koderover/zadig/pkg/tool/log" ) func GetServiceRenderCharts(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - if c.Query("productName") == "" { + if c.Query("projectName") == "" { ctx.Err = e.ErrInvalidParam.AddDesc("productName can not be null!") return } @@ -42,14 +41,14 @@ func GetServiceRenderCharts(c *gin.Context) { return } - ctx.Resp, ctx.Err = service.GetRenderCharts(c.Query("productName"), c.Query("envName"), c.Query("serviceName"), ctx.Logger) + ctx.Resp, ctx.Err = commonservice.GetRenderCharts(c.Query("projectName"), c.Query("envName"), c.Query("serviceName"), ctx.Logger) } -func CreateOrUpdateRenderChart(c *gin.Context) { +func GetProductDefaultValues(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - if c.Query("productName") == "" { + if c.Query("projectName") == "" { ctx.Err = e.ErrInvalidParam.AddDesc("productName can not be null!") return } @@ -59,17 +58,29 @@ func CreateOrUpdateRenderChart(c *gin.Context) { return } - data, err := c.GetRawData() - if err != nil { - log.Errorf("CreateOrUpdateRenderChart c.GetRawData() err : %v", err) + ctx.Resp, ctx.Err = service.GetDefaultValues(c.Query("projectName"), c.Query("envName"), ctx.Logger) +} + +func GetYamlContent(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + arg := &service.YamlContentRequestArg{} + + if err := c.ShouldBindQuery(arg); err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + if arg.CodehostID == 0 && len(arg.RepoLink) == 0 { + ctx.Err = e.ErrInvalidParam.AddDesc("neither codehost nor repo link is specified") + return } - args := new(commonservice.RenderChartArg) - if err = json.Unmarshal(data, args); err != nil { - log.Errorf("CreateOrUpdateRenderChart json.Unmarshal err : %v", err) - ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + if len(arg.ValuesPaths) == 0 { + ctx.Err = e.ErrInvalidParam.AddDesc("paths can't be empty") + return } - internalhandler.InsertOperationLog(c, ctx.Username, c.Param("productName"), "新增", "环境变量", c.Query("envName"), string(data), ctx.Logger) - ctx.Err = service.CreateOrUpdateChartValues(c.Query("productName"), c.Query("envName"), args, ctx.Username, ctx.RequestID, ctx.Logger) + pathArr := strings.Split(arg.ValuesPaths, ",") + ctx.Resp, ctx.Err = service.GetMergedYamlContent(arg, pathArr) } diff --git a/pkg/microservice/aslan/core/environment/handler/revision.go b/pkg/microservice/aslan/core/environment/handler/revision.go index 77ed70d34aa6ab5e1212bb4ec7b74ae8324c34bf..c8cba110975f1d7fcd04e7a15a4e336efe5f70c1 100644 --- a/pkg/microservice/aslan/core/environment/handler/revision.go +++ b/pkg/microservice/aslan/core/environment/handler/revision.go @@ -28,9 +28,10 @@ func ListProductsRevision(c *gin.Context) { defer func() { internalhandler.JSONResponse(c, ctx) }() // trigger by cron service trigger := c.Query("basicFacility") - if trigger != "" { - ctx.Resp, ctx.Err = service.ListProductsRevisionByFacility(c.Query("basicFacility"), ctx.Logger) + deployType := c.Query("deployType") + if trigger != "" || deployType != "" { + ctx.Resp, ctx.Err = service.ListProductsRevisionByOption(trigger, deployType, ctx.Logger) return } - ctx.Resp, ctx.Err = service.ListProductsRevision(c.Query("productName"), c.Query("envName"), ctx.User.ID, ctx.User.IsSuperUser, ctx.Logger) + ctx.Resp, ctx.Err = service.ListProductsRevision(c.Query("projectName"), c.Query("envName"), ctx.Logger) } diff --git a/pkg/microservice/aslan/core/environment/handler/router.go b/pkg/microservice/aslan/core/environment/handler/router.go index 444bf3358966acc2fa9f75b9088de8564c4f2faa..2cf5f60c2349f6ce7f7fbc9fc3363da8ba357b05 100644 --- a/pkg/microservice/aslan/core/environment/handler/router.go +++ b/pkg/microservice/aslan/core/environment/handler/router.go @@ -18,24 +18,41 @@ package handler import ( "github.com/gin-gonic/gin" - - gin2 "github.com/koderover/zadig/pkg/middleware/gin" - "github.com/koderover/zadig/pkg/types/permission" ) type Router struct{} func (*Router) Inject(router *gin.RouterGroup) { - router.Use(gin2.Auth()) - // --------------------------------------------------------------------------------------- // Kube配置管理接口 ConfigMap // --------------------------------------------------------------------------------------- configmaps := router.Group("configmaps") { - configmaps.GET("", ListConfigMaps) - configmaps.PUT("", gin2.StoreProductName, gin2.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, UpdateConfigMap) - configmaps.POST("", gin2.StoreProductName, gin2.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, RollBackConfigMap) + configmaps.GET("/:envName", ListConfigMaps) + configmaps.POST("", RollBackConfigMap) + configmaps.GET("/migrate", MigrateHistoryConfigMaps) + } + + secrets := router.Group("secrets") + { + secrets.GET("/:envName", ListSecrets) + } + ingresses := router.Group("ingresses") + { + ingresses.GET("/:envName", ListIngresses) + } + pvcs := router.Group("pvcs") + { + pvcs.GET("/:envName", ListPvcs) + } + commonEnvCfgs := router.Group("envcfgs") + { + commonEnvCfgs.GET("/:envName/cfg/:objectName", ListCommonEnvCfgHistory) + commonEnvCfgs.GET("", ListLatestEnvCfg) + commonEnvCfgs.PUT("/:envName/:type/:objectName/sync", SyncEnvResource) + commonEnvCfgs.PUT("/:envName", UpdateCommonEnvCfg) + commonEnvCfgs.POST("/:envName", CreateCommonEnvCfg) + commonEnvCfgs.DELETE("/:envName/cfg/:objectName", DeleteCommonEnvCfg) } // --------------------------------------------------------------------------------------- @@ -60,7 +77,7 @@ func (*Router) Inject(router *gin.RouterGroup) { // --------------------------------------------------------------------------------------- export := router.Group("export") { - export.GET("/service", gin2.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID}, permission.QueryType), ExportYaml) + export.GET("/service", ExportYaml) // export.GET("/pipelines/:name", ExportBuildYaml) } @@ -69,8 +86,8 @@ func (*Router) Inject(router *gin.RouterGroup) { // --------------------------------------------------------------------------------------- image := router.Group("image") { - image.POST("/deployment", gin2.StoreProductName, gin2.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID, permission.TestUpdateEnvUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, UpdateDeploymentContainerImage) - image.POST("/statefulset", gin2.StoreProductName, gin2.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID, permission.TestUpdateEnvUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, UpdateStatefulSetContainerImage) + image.POST("/deployment/:envName", UpdateDeploymentContainerImage) + image.POST("/statefulset/:envName", UpdateStatefulSetContainerImage) } // 查询环境创建时的服务和变量信息 @@ -86,54 +103,62 @@ func (*Router) Inject(router *gin.RouterGroup) { kube.GET("/events", ListKubeEvents) kube.POST("/pods", ListServicePods) - kube.DELETE("/pods/:podName", gin2.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID, permission.TestUpdateEnvUUID}, permission.QueryType), gin2.UpdateOperationLogStatus, DeletePod) + kube.DELETE("/:env/pods/:podName", DeletePod) kube.GET("/pods/:podName/events", ListPodEvents) kube.GET("/workloads", ListWorkloads) + kube.GET("/nodes", ListNodes) + + kube.POST("/:env/pods/:podName/debugcontainer", PatchDebugContainer) } // --------------------------------------------------------------------------------------- - // 产品管理接口(集成环境) + // 产品管理接口(环境) // --------------------------------------------------------------------------------------- environments := router.Group("environments") { environments.GET("", ListProducts) - environments.POST("/:productName/auto", gin2.IsHavePermission([]string{permission.TestEnvCreateUUID}, permission.ParamType), AutoCreateProduct) - environments.PUT("/:productName/autoUpdate", gin2.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, AutoUpdateProduct) - environments.POST("", gin2.StoreProductName, gin2.IsHavePermission([]string{permission.TestEnvCreateUUID, permission.ProdEnvCreateUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, CreateProduct) - - environments.POST("/:productName/helm", gin2.StoreProductName, gin2.IsHavePermission([]string{permission.TestEnvCreateUUID, permission.ProdEnvCreateUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, CreateHelmProduct) - environments.PUT("/:productName/multiHelmEnv", gin2.StoreProductName, gin2.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, UpdateMultiHelmEnv) - - environments.POST("/:productName", gin2.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, UpdateProduct) - environments.PUT("/:productName/envRecycle", gin2.IsHavePermission([]string{permission.TestEnvManageUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, UpdateProductRecycleDay) - environments.PUT("/:productName/renderchart", gin2.IsHavePermission([]string{permission.TestEnvManageUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, UpdateHelmProductRenderCharts) - - environments.GET("/:productName/helmChartVersions", GetHelmChartVersions) - environments.PUT("/:productName", gin2.IsHavePermission([]string{permission.TestEnvShareUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, EnvShare) - - environments.GET("/:productName", GetProduct) - environments.GET("/:productName/productInfo", GetProductInfo) - environments.GET("/:productName/ingressInfo", GetProductIngress) - environments.GET("/:productName/helmRenderCharts", ListRenderCharts) - environments.DELETE("/:productName", gin2.IsHavePermission([]string{permission.TestEnvDeleteUUID, permission.ProdEnvDeleteUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, DeleteProduct) - - environments.GET("/:productName/groups", ListGroups) - environments.GET("/:productName/workloads", ListWorkloadsInEnv) - - environments.GET("/:productName/services/:serviceName", GetService) - environments.PUT("/:productName/services/:serviceName/:serviceType", gin2.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, UpdateService) - environments.POST("/:productName/services/:serviceName/restart", gin2.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, RestartService) - environments.POST("/:productName/services/:serviceName/restartNew", gin2.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID, permission.TestUpdateEnvUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, RestartNewService) - environments.POST("/:productName/services/:serviceName/scale/:number", gin2.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, ScaleService) - environments.POST("/:productName/services/:serviceName/scaleNew/:number", gin2.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, ScaleNewService) - environments.GET("/:productName/services/:serviceName/containers/:container/namespaces/:namespace", GetServiceContainer) - - environments.GET("/estimated-renderchart", GetEstimatedRenderCharts) - - // need to be deprecated - environments.PUT("/:productName/helmEnv", gin2.IsHavePermission([]string{permission.TestEnvManageUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, UpdateHelmProduct) - environments.PUT("/:productName/helmEnvVariable", gin2.IsHavePermission([]string{permission.TestEnvManageUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, UpdateHelmProductVariable) - environments.PUT("/:productName/updateMultiEnv", gin2.IsHavePermission([]string{permission.TestEnvManageUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, UpdateMultiHelmProduct) + environments.PUT("/:name", UpdateProduct) + environments.PUT("/:name/registry", UpdateProductRegistry) + environments.PUT("", UpdateMultiProducts) + environments.POST("", CreateProduct) + environments.GET("/:name", GetProduct) + environments.PUT("/:name/envRecycle", UpdateProductRecycleDay) + environments.POST("/:name/estimated-values", EstimatedValues) + environments.PUT("/:name/renderset", UpdateHelmProductRenderset) + environments.PUT("/:name/helm/default-values", UpdateHelmProductDefaultValues) + environments.PUT("/:name/helm/charts", UpdateHelmProductCharts) + environments.PUT("/:name/syncVariables", SyncHelmProductRenderset) + environments.GET("/:name/helmChartVersions", GetHelmChartVersions) + environments.GET("/:name/productInfo", GetProductInfo) + environments.DELETE("/:name", DeleteProduct) + environments.PUT("/:name/services", DeleteProductServices) + environments.GET("/:name/groups", ListGroups) + environments.GET("/:name/workloads", ListWorkloadsInEnv) + + environments.GET("/:name/helm/releases", ListReleases) + environments.GET("/:name/helm/values", GetChartValues) + environments.GET("/:name/helm/charts", GetChartInfos) + environments.GET("/:name/helm/images", GetImageInfos) + + environments.GET("/:name/services/:serviceName", GetService) + environments.PUT("/:name/services/:serviceName", UpdateService) + environments.POST("/:name/services/:serviceName/restart", RestartService) + environments.POST("/:name/services/:serviceName/restartNew", RestartNewService) + environments.POST("/:name/services/:serviceName/scale", ScaleService) + environments.POST("/:name/services/:serviceName/scaleNew", ScaleNewService) + environments.GET("/:name/services/:serviceName/containers/:container", GetServiceContainer) + + environments.GET("/:name/estimated-renderchart", GetEstimatedRenderCharts) + + environments.GET("/:name/check/workloads/k8services", CheckWorkloadsK8sServices) + environments.POST("/:name/share/enable", EnableBaseEnv) + environments.DELETE("/:name/share/enable", DisableBaseEnv) + environments.GET("/:name/check/sharenv/:op/ready", CheckShareEnvReady) + + environments.GET("/:name/services/:serviceName/pmexec", ConnectSshPmExec) + + environments.POST("/:name/services/:serviceName/devmode/patch", PatchWorkload) + environments.POST("/:name/services/:serviceName/devmode/recover", RecoverWorkload) } // --------------------------------------------------------------------------------------- @@ -142,7 +167,8 @@ func (*Router) Inject(router *gin.RouterGroup) { rendersets := router.Group("rendersets") { rendersets.GET("/renderchart", GetServiceRenderCharts) - rendersets.PUT("/renderchart", CreateOrUpdateRenderChart) // create or update renderchart when onBoarding + rendersets.GET("/default-values", GetProductDefaultValues) + rendersets.GET("/yamlContent", GetYamlContent) } // --------------------------------------------------------------------------------------- @@ -152,4 +178,9 @@ func (*Router) Inject(router *gin.RouterGroup) { { revision.GET("/products", ListProductsRevision) } + + bundles := router.Group("bundle-resources") + { + bundles.GET("", GetBundleResources) + } } diff --git a/pkg/microservice/aslan/core/environment/handler/secret.go b/pkg/microservice/aslan/core/environment/handler/secret.go new file mode 100644 index 0000000000000000000000000000000000000000..3e97633a3f0cb2fb4f5446ee0b6e7a552751b6f3 --- /dev/null +++ b/pkg/microservice/aslan/core/environment/handler/secret.go @@ -0,0 +1,38 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +func ListSecrets(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + envName := c.Param("envName") + productName := c.Query("projectName") + if envName == "" || productName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("param envName or projectName is invalid") + return + } + + ctx.Resp, ctx.Err = service.ListSecrets(envName, productName, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/environment/handler/service.go b/pkg/microservice/aslan/core/environment/handler/service.go index 6c5cffd871897aa10868c4387a22f18a0c78a11d..34bd1a910dadc51f892f61d3acc24fb090d5e6a3 100644 --- a/pkg/microservice/aslan/core/environment/handler/service.go +++ b/pkg/microservice/aslan/core/environment/handler/service.go @@ -31,12 +31,12 @@ import ( func GetService(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - envName := c.Query("envName") - productName := c.Param("productName") + envName := c.Param("name") + projectName := c.Query("projectName") serviceName := c.Param("serviceName") workLoadType := c.Query("workLoadType") - ctx.Resp, ctx.Err = service.GetService(envName, productName, serviceName, workLoadType, ctx.Logger) + ctx.Resp, ctx.Err = service.GetService(envName, projectName, serviceName, workLoadType, ctx.Logger) } func RestartService(c *gin.Context) { @@ -44,19 +44,22 @@ func RestartService(c *gin.Context) { defer func() { internalhandler.JSONResponse(c, ctx) }() args := &service.SvcOptArgs{ - EnvName: c.Query("envName"), - ProductName: c.Param("productName"), + EnvName: c.Param("name"), + ProductName: c.Query("projectName"), ServiceName: c.Param("serviceName"), } - internalhandler.InsertOperationLog(c, ctx.Username, c.Param("productName"), "重启", "集成环境-服务", fmt.Sprintf("环境名称:%s,服务名称:%s", c.Query("envName"), c.Param("serviceName")), "", ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, c.Query("projectName"), "重启", "环境-服务", fmt.Sprintf("环境名称:%s,服务名称:%s", c.Param("name"), c.Param("serviceName")), "", ctx.Logger) ctx.Err = service.RestartService(args.EnvName, args, ctx.Logger) } func UpdateService(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - internalhandler.InsertOperationLog(c, ctx.Username, c.Param("productName"), "更新", "集成环境-单服务", fmt.Sprintf("环境名称:%s,服务名称:%s", c.Query("envName"), c.Param("serviceName")), "", ctx.Logger) + + envName := c.Param("name") + projectName := c.Query("projectName") + internalhandler.InsertOperationLog(c, ctx.UserName, projectName, "更新", "环境-单服务", fmt.Sprintf("环境名称:%s,服务名称:%s", envName, c.Param("serviceName")), "", ctx.Logger) svcRev := new(service.SvcRevision) if err := c.BindJSON(svcRev); err != nil { @@ -65,12 +68,12 @@ func UpdateService(c *gin.Context) { } args := &service.SvcOptArgs{ - EnvName: c.Query("envName"), - ProductName: c.Param("productName"), + EnvName: envName, + ProductName: projectName, ServiceName: c.Param("serviceName"), - ServiceType: c.Param("serviceType"), + ServiceType: c.Query("serviceType"), ServiceRev: svcRev, - UpdateBy: ctx.Username, + UpdateBy: ctx.UserName, } ctx.Err = service.UpdateService(args, ctx.Logger) @@ -81,18 +84,18 @@ func RestartNewService(c *gin.Context) { defer func() { internalhandler.JSONResponse(c, ctx) }() args := &service.RestartScaleArgs{ - EnvName: c.Query("envName"), - ProductName: c.Param("productName"), + EnvName: c.Param("name"), + ProductName: c.Query("projectName"), ServiceName: c.Param("serviceName"), Type: c.Query("type"), Name: c.Query("name"), } internalhandler.InsertOperationLog( - c, ctx.Username, - c.Param("productName"), + c, ctx.UserName, + c.Query("projectName"), "重启", - "集成环境-服务", + "环境-服务", fmt.Sprintf( "环境名称:%s,服务名称:%s,%s:%s", args.EnvName, args.ServiceName, args.Type, args.Name, ), @@ -109,21 +112,21 @@ func ScaleNewService(c *gin.Context) { args := new(service.ScaleArgs) args.Type = setting.Deployment - productName := c.Param("productName") + projectName := c.Query("projectName") serviceName := c.Param("serviceName") - envName := c.Query("envName") + envName := c.Param("name") resourceType := c.Query("type") name := c.Query("name") internalhandler.InsertOperationLog( - c, ctx.Username, - c.Param("productName"), + c, ctx.UserName, + projectName, "伸缩", - "集成环境-服务", - fmt.Sprintf("环境名称:%s,%s:%s", args.EnvName, args.Type, args.Name), + "环境-服务", + fmt.Sprintf("环境名称:%s,%s:%s", envName, resourceType, name), "", ctx.Logger) - number, err := strconv.Atoi(c.Param("number")) + number, err := strconv.Atoi(c.Query("number")) if err != nil { ctx.Err = e.ErrInvalidParam.AddDesc("invalid number format") return @@ -131,7 +134,7 @@ func ScaleNewService(c *gin.Context) { ctx.Err = service.Scale(&service.ScaleArgs{ Type: resourceType, - ProductName: productName, + ProductName: projectName, EnvName: envName, ServiceName: serviceName, Name: name, @@ -142,21 +145,22 @@ func ScaleNewService(c *gin.Context) { func ScaleService(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - internalhandler.InsertOperationLog(c, ctx.Username, c.Param("productName"), "伸缩", "集成环境-服务", fmt.Sprintf("环境名称:%s,服务名称:%s", c.Query("envName"), c.Param("serviceName")), "", ctx.Logger) - number, err := strconv.Atoi(c.Param("number")) + envName := c.Param("name") + projectName := c.Query("projectName") + internalhandler.InsertOperationLog(c, ctx.UserName, projectName, "伸缩", "环境-服务", fmt.Sprintf("环境名称:%s,服务名称:%s", envName, c.Param("serviceName")), "", ctx.Logger) + + number, err := strconv.Atoi(c.Query("number")) if err != nil { ctx.Err = e.ErrInvalidParam.AddDesc("invalid number format") return } - productName := c.Param("productName") serviceName := c.Param("serviceName") - envName := c.Query("envName") ctx.Err = service.ScaleService( envName, - productName, + projectName, serviceName, number, ctx.Logger, @@ -167,11 +171,10 @@ func GetServiceContainer(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - envName := c.Query("envName") - //namespace := c.Param("namespace") - productName := c.Param("productName") + envName := c.Param("name") + projectName := c.Query("projectName") serviceName := c.Param("serviceName") container := c.Param("container") - ctx.Err = service.GetServiceContainer(envName, productName, serviceName, container, ctx.Logger) + ctx.Err = service.GetServiceContainer(envName, projectName, serviceName, container, ctx.Logger) } diff --git a/pkg/microservice/aslan/core/environment/handler/share_env.go b/pkg/microservice/aslan/core/environment/handler/share_env.go new file mode 100644 index 0000000000000000000000000000000000000000..a9812213797e81ed02d2ae22dca94ff4afe837b9 --- /dev/null +++ b/pkg/microservice/aslan/core/environment/handler/share_env.go @@ -0,0 +1,56 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" +) + +func CheckWorkloadsK8sServices(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.CheckWorkloadsK8sServices(c, c.Param("name"), c.Query("projectName")) +} + +func EnableBaseEnv(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + internalhandler.InsertOperationLog(c, ctx.UserName, c.Query("projectName"), "开启自测模式", "环境", c.Param("name"), "", ctx.Logger) + + ctx.Err = service.EnableBaseEnv(c, c.Param("name"), c.Query("projectName")) +} + +func DisableBaseEnv(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + internalhandler.InsertOperationLog(c, ctx.UserName, c.Query("projectName"), "关闭自测模式", "环境", c.Param("name"), "", ctx.Logger) + + ctx.Err = service.DisableBaseEnv(c, c.Param("name"), c.Query("projectName")) +} + +func CheckShareEnvReady(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.CheckShareEnvReady(c, c.Param("name"), c.Param("op"), c.Query("projectName")) +} diff --git a/pkg/microservice/aslan/core/environment/service/bundle.go b/pkg/microservice/aslan/core/environment/service/bundle.go new file mode 100644 index 0000000000000000000000000000000000000000..405a50a3e0739cb03d606e8b686735564974ea8f --- /dev/null +++ b/pkg/microservice/aslan/core/environment/service/bundle.go @@ -0,0 +1,99 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "strconv" + + "go.uber.org/zap" + + commonConfig "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/label/config" + labeldb "github.com/koderover/zadig/pkg/microservice/aslan/core/label/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/label/service" + "github.com/koderover/zadig/pkg/tool/log" +) + +type resourceSpec struct { + ResourceID string `json:"resourceID"` + ProjectName string `json:"projectName"` + Spec []string `json:"spec"` +} + +func GetBundleResources(logger *zap.SugaredLogger) ([]*resourceSpec, error) { + var res []*resourceSpec + + envs, err := mongodb.NewProductColl().List(nil) + if err != nil { + logger.Errorf("Failed to list envs, err: %s", err) + return nil, err + } + + // get labels by workflow resources ids + var resources []labeldb.Resource + for _, env := range envs { + resource := labeldb.Resource{ + Name: env.EnvName, + ProjectName: env.ProductName, + Type: string(config.ResourceTypeEnvironment), + } + resources = append(resources, resource) + } + labelsResp, err := service.ListLabelsByResources(resources, logger) + if err != nil { + logger.Errorf("ListLabelsByResources err:%s", err) + return nil, err + } + + // production attribute + clusterMap := make(map[string]*models.K8SCluster) + clusters, err := mongodb.NewK8SClusterColl().List(nil) + if err != nil { + log.Errorf("Failed to list clusters in db, err: %s", err) + return nil, err + } + + for _, cls := range clusters { + clusterMap[cls.ID.Hex()] = cls + } + + for _, env := range envs { + resourceKey := commonConfig.BuildResourceKey(string(config.ResourceTypeEnvironment), env.ProductName, env.EnvName) + resourceSpec := &resourceSpec{ + ResourceID: env.EnvName, + ProjectName: env.ProductName, + } + if labels, ok := labelsResp.Labels[resourceKey]; ok { + for _, v := range labels { + resourceSpec.Spec = append(resourceSpec.Spec, v.Key+":"+v.Value) + } + } + + clusterID := env.ClusterID + production := false + cluster, ok := clusterMap[clusterID] + if ok { + production = cluster.Production + } + resourceSpec.Spec = append(resourceSpec.Spec, "production:"+strconv.FormatBool(production)) + res = append(res, resourceSpec) + } + + return res, nil +} diff --git a/pkg/microservice/aslan/core/environment/service/common_env_cfg.go b/pkg/microservice/aslan/core/environment/service/common_env_cfg.go new file mode 100644 index 0000000000000000000000000000000000000000..507837b72c56fa4a1f8246d6849ba6786938aeb7 --- /dev/null +++ b/pkg/microservice/aslan/core/environment/service/common_env_cfg.go @@ -0,0 +1,721 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "encoding/json" + "fmt" + "time" + + "go.mongodb.org/mongo-driver/bson/primitive" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + templatemodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" + fsservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/fs" + "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/kube/getter" + "github.com/koderover/zadig/pkg/tool/kube/serializer" + "github.com/koderover/zadig/pkg/tool/kube/updater" + "github.com/koderover/zadig/pkg/tool/log" + yamlutil "github.com/koderover/zadig/pkg/util/yaml" +) + +type ResourceWithLabel interface { + GetLabels() map[string]string + SetLabels(labels map[string]string) + SetNamespace(ns string) +} + +type ResourceResponseBase struct { + Name string `json:"-"` + Type config.CommonEnvCfgType `json:"-"` + EnvName string `json:"-"` + ProjectName string `json:"-"` + YamlData string `json:"yaml_data"` + UpdateUserName string `json:"update_username"` + CreateTime time.Time `json:"create_time"` + Services []string `json:"services"` + SourceDetail *models.CreateFromRepo `json:"source_detail"` + AutoSync bool `json:"auto_sync"` +} + +func (resp *ResourceResponseBase) setSourceDetailData(resource ResourceWithLabel) { + envResource, err := commonrepo.NewEnvResourceColl().Find(&commonrepo.QueryEnvResourceOption{ + Name: resp.Name, + Type: string(resp.Type), + EnvName: resp.EnvName, + ProductName: resp.ProjectName, + IgnoreNotFound: true, + Active: true, + }) + if err != nil { + log.Warnf("failed to f get %v with name : %s, err: %s", string(resp.Type), resp.Name, err) + return + } + if envResource != nil { + resp.SourceDetail = envResource.SourceDetail + resp.AutoSync = envResource.AutoSync + resp.UpdateUserName = envResource.UpdateUserName + } +} + +func DeleteCommonEnvCfg(envName, productName, objectName string, commonEnvCfgType config.CommonEnvCfgType, log *zap.SugaredLogger) error { + product, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: productName, + EnvName: envName, + }) + if err != nil { + return e.ErrDeleteResource.AddErr(err) + } + + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), product.ClusterID) + if err != nil { + return e.ErrDeleteResource.AddErr(err) + } + clientset, err := kubeclient.GetKubeClientSet(config.HubServerAddress(), product.ClusterID) + if err != nil { + log.Errorf("failed to create kubernetes clientset for clusterID: %s, the error is: %s", product.ClusterID, err) + return e.ErrDeleteResource.AddErr(err) + } + + switch commonEnvCfgType { + case config.CommonEnvCfgTypeConfigMap: + err = updater.DeleteConfigMap(product.Namespace, objectName, kubeClient) + case config.CommonEnvCfgTypeSecret: + err = updater.DeleteSecretWithName(product.Namespace, objectName, kubeClient) + case config.CommonEnvCfgTypeIngress: + err = updater.DeleteIngresseWithName(product.Namespace, objectName, clientset) + case config.CommonEnvCfgTypePvc: + err = updater.DeletePvcWithName(product.Namespace, objectName, clientset) + default: + return e.ErrDeleteResource.AddDesc(fmt.Sprintf("%s is not support delete", commonEnvCfgType)) + } + if err != nil { + log.Error(err) + return e.ErrDeleteResource.AddDesc(err.Error()) + } + + resourceData, err := commonrepo.NewEnvResourceColl().Find(&commonrepo.QueryEnvResourceOption{ + ProductName: productName, + EnvName: product.EnvName, + Name: objectName, + Type: string(commonEnvCfgType), + IgnoreNotFound: true, + }) + if err != nil { + return err + } + + if resourceData != nil { + return commonrepo.NewEnvResourceColl().Delete(resourceData.ID) + } + return nil +} + +// ensureLabelAndNs ensure label 's-product' exists in particular resource(secret/configmap) deployed by zadig +func ensureLabelAndNs(res ResourceWithLabel, namespace, projectName string) (string, error) { + resLabels := res.GetLabels() + if resLabels == nil { + resLabels = make(map[string]string) + } + resLabels[setting.ProductLabel] = projectName + res.SetLabels(resLabels) + res.SetNamespace(namespace) + + jsonBytes, err := json.Marshal(res) + if err != nil { + return "", err + } + yamlBytes, err := yaml.JSONToYAML(jsonBytes) + return string(yamlBytes), err +} + +func geneSourceDetail(gitRepoConfig *templatemodels.GitRepoConfig) *models.CreateFromRepo { + if gitRepoConfig == nil { + return nil + } + ret := &models.CreateFromRepo{ + GitRepoConfig: &templatemodels.GitRepoConfig{ + CodehostID: gitRepoConfig.CodehostID, + Owner: gitRepoConfig.Owner, + Repo: gitRepoConfig.Repo, + Branch: gitRepoConfig.Branch, + Namespace: gitRepoConfig.GetNamespace(), + }, + } + if len(gitRepoConfig.ValuesPaths) > 0 { + ret.LoadPath = gitRepoConfig.ValuesPaths[0] + } + return ret +} + +func getResourceYamlAndName(sourceYaml []byte, namespace, productName string, resType config.CommonEnvCfgType) (name, yamlData string, err error) { + switch resType { + case config.CommonEnvCfgTypeConfigMap: + cm := &corev1.ConfigMap{} + err = json.Unmarshal(sourceYaml, cm) + if err != nil { + return + } + yamlData, err = ensureLabelAndNs(cm, namespace, productName) + if err != nil { + return + } + name = cm.Name + case config.CommonEnvCfgTypeSecret: + secret := &corev1.Secret{} + err = json.Unmarshal(sourceYaml, secret) + if err != nil { + return + } + yamlData, err = ensureLabelAndNs(secret, namespace, productName) + if err != nil { + return + } + name = secret.Name + case config.CommonEnvCfgTypeIngress: + u, errDecode := serializer.NewDecoder().YamlToUnstructured(sourceYaml) + if errDecode != nil { + err = fmt.Errorf("Failed to convert yaml to Unstructured, manifest is\n%s\n, error: %v", string(sourceYaml), errDecode) + return + } + yamlData, err = ensureLabelAndNs(u, namespace, productName) + if err != nil { + return + } + name = u.GetName() + case config.CommonEnvCfgTypePvc: + pvc := &corev1.PersistentVolumeClaim{} + err = json.Unmarshal(sourceYaml, pvc) + if err != nil { + return + } + yamlData, err = ensureLabelAndNs(pvc, namespace, productName) + if err != nil { + return + } + name = pvc.Name + default: + err = fmt.Errorf("%s is not support create", resType) + return + } + return +} + +func CreateCommonEnvCfg(args *models.CreateUpdateCommonEnvCfgArgs, userName string, log *zap.SugaredLogger) error { + js, err := yaml.YAMLToJSON([]byte(args.YamlData)) + if err != nil { + return e.ErrUpdateResource.AddErr(err) + } + + product, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: args.ProductName, + EnvName: args.EnvName, + }) + if err != nil { + return e.ErrUpdateResource.AddErr(err) + } + + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), product.ClusterID) + if err != nil { + return e.ErrUpdateResource.AddErr(err) + } + clientset, err := kubeclient.GetKubeClientSet(config.HubServerAddress(), product.ClusterID) + if err != nil { + log.Errorf("failed to create kubernetes clientset for clusterID: %s, the error is: %s", product.ClusterID, err) + return e.ErrUpdateResource.AddErr(err) + } + + u, err := serializer.NewDecoder().YamlToUnstructured(js) + if err != nil { + return e.ErrUpdateResource.AddErr(err) + } + if args.CommonEnvCfgType == config.CommonEnvCfgTypePvc { + if u.GetKind() != setting.PersistentVolumeClaim { + return e.ErrUpdateResource.AddDesc(fmt.Sprintf("param commonEnvCfgType:%s not match yaml kind %s ", args.CommonEnvCfgType, u.GetKind())) + } + } else if u.GetKind() != string(args.CommonEnvCfgType) { + return e.ErrUpdateResource.AddDesc(fmt.Sprintf("param commonEnvCfgType:%s not match yaml kind %s ", args.CommonEnvCfgType, u.GetKind())) + } + + envResource := &models.EnvResource{ + ProductName: args.ProductName, + UpdateUserName: userName, + EnvName: args.EnvName, + Namespace: product.Namespace, + Type: string(args.CommonEnvCfgType), + SourceDetail: geneSourceDetail(args.GitRepoConfig), + AutoSync: args.AutoSync, + } + + switch args.CommonEnvCfgType { + case config.CommonEnvCfgTypeConfigMap: + cm := &corev1.ConfigMap{} + err = json.Unmarshal(js, cm) + if err != nil { + return e.ErrUpdateResource.AddErr(err) + } + yamlData, err := ensureLabelAndNs(cm, product.Namespace, args.ProductName) + if err != nil { + return e.ErrUpdateResource.AddErr(err) + } + + if err := updater.CreateConfigMap(cm, kubeClient); err != nil { + log.Error(err) + return e.ErrUpdateResource.AddErr(err) + } + envResource.Name = cm.Name + envResource.YamlData = yamlData + + case config.CommonEnvCfgTypeSecret: + secret := &corev1.Secret{} + err = json.Unmarshal(js, secret) + if err != nil { + return e.ErrUpdateResource.AddErr(err) + } + yamlData, err := ensureLabelAndNs(secret, product.Namespace, args.ProductName) + if err != nil { + return e.ErrUpdateResource.AddErr(err) + } + + if err := updater.UpdateOrCreateSecret(secret, kubeClient); err != nil { + log.Error(err) + return e.ErrUpdateResource.AddDesc(err.Error()) + } + envResource.Name = secret.Name + envResource.YamlData = yamlData + + case config.CommonEnvCfgTypeIngress: + u, err := serializer.NewDecoder().YamlToUnstructured([]byte(args.YamlData)) + if err != nil { + log.Errorf("Failed to convert yaml to Unstructured, manifest is\n%s\n, error: %v", args.YamlData, err) + return e.ErrUpdateResource.AddDesc("ingress Yaml Name is incorrect") + } + + yamlData, err := ensureLabelAndNs(u, product.Namespace, args.ProductName) + if err != nil { + return e.ErrUpdateResource.AddErr(err) + } + + err = updater.UpdateOrCreateUnstructured(u, kubeClient) + if err != nil { + log.Errorf("Failed to UpdateOrCreateIngress %s, manifest is\n%v\n, error: %v", u.GetKind(), u, err) + return e.ErrUpdateResource.AddErr(fmt.Errorf("Failed to UpdateOrCreateIngress %s, manifest is\n%v\n, error: %v", u.GetKind(), u, err)) + } + envResource.Name = u.GetName() + envResource.YamlData = yamlData + + case config.CommonEnvCfgTypePvc: + pvc := &corev1.PersistentVolumeClaim{} + err = json.Unmarshal(js, pvc) + if err != nil { + return e.ErrUpdateResource.AddErr(err) + } + + yamlData, err := ensureLabelAndNs(pvc, product.Namespace, args.ProductName) + if err != nil { + return e.ErrUpdateResource.AddErr(err) + } + + if err := updater.CreatePvc(product.Namespace, pvc, clientset); err != nil { + log.Error(err) + return e.ErrUpdateResource.AddErr(err) + } + envResource.Name = pvc.Name + envResource.YamlData = yamlData + + default: + return e.ErrUpdateResource.AddDesc(fmt.Sprintf("%s is not support create", args.CommonEnvCfgType)) + } + + if err = commonrepo.NewEnvResourceColl().Create(envResource); err != nil { + return e.ErrUpdateResource.AddDesc(err.Error()) + } + + return nil +} + +func getLatestEnvResource(name, resType, envName, productName string) (*models.EnvResource, error) { + return commonrepo.NewEnvResourceColl().Find( + &commonrepo.QueryEnvResourceOption{ + Name: name, + ProductName: productName, + Type: resType, + EnvName: envName, + IsSort: true, + }) +} + +func UpdateCommonEnvCfg(args *models.CreateUpdateCommonEnvCfgArgs, userName string, updateContentOnly bool, log *zap.SugaredLogger) error { + var err error + u, err := serializer.NewDecoder().YamlToUnstructured([]byte(args.YamlData)) + if err != nil { + return e.ErrUpdateResource.AddErr(err) + } + if args.CommonEnvCfgType == config.CommonEnvCfgTypePvc { + if u.GetKind() != setting.PersistentVolumeClaim { + return e.ErrUpdateResource.AddDesc(fmt.Sprintf("param commonEnvCfgType:%s not match yaml kind %s ", args.CommonEnvCfgType, u.GetKind())) + } + } else if u.GetKind() != string(args.CommonEnvCfgType) { + return e.ErrUpdateResource.AddDesc(fmt.Sprintf("param commonEnvCfgType:%s not match yaml kind %s ", args.CommonEnvCfgType, u.GetKind())) + } + + // when rollback/autoSync env configs, git config should not change + if updateContentOnly { + if args.LatestEnvResource == nil { + args.LatestEnvResource, err = getLatestEnvResource(args.Name, string(args.CommonEnvCfgType), args.EnvName, args.ProductName) + } + if err != nil { + return err + } + if args.LatestEnvResource == nil { + return fmt.Errorf("failed to find env resource %s:%s", args.Name, string(args.CommonEnvCfgType)) + } + args.AutoSync = args.LatestEnvResource.AutoSync + args.SourceDetail = args.LatestEnvResource.SourceDetail + } else { + args.SourceDetail = geneSourceDetail(args.GitRepoConfig) + } + + switch args.CommonEnvCfgType { + case config.CommonEnvCfgTypeConfigMap: + err = UpdateConfigMap(args, userName, log) + case config.CommonEnvCfgTypeSecret: + err = UpdateSecret(args, userName, log) + case config.CommonEnvCfgTypeIngress: + err = UpdateOrCreateIngress(args, userName, false, log) + case config.CommonEnvCfgTypePvc: + err = UpdatePvc(args, userName, log) + default: + return e.ErrUpdateResource.AddDesc(fmt.Sprintf("%s is not support update", args.CommonEnvCfgType)) + } + if err != nil { + log.Error(err) + return err + } + return nil +} + +type ListCommonEnvCfgHistoryArgs struct { + EnvName string `json:"env_name" form:"envName"` + ProjectName string `json:"project_name" form:"projectName"` + Name string `json:"name" form:"name"` + CommonEnvCfgType config.CommonEnvCfgType `json:"common_env_cfg_type" form:"commonEnvCfgType"` + AutoSync bool `json:"auto_sync" form:"autoSync"` +} + +type SyncEnvResourceArg struct { + EnvName string `json:"env_name"` + ProductName string `json:"product_name"` + Name string `json:"name"` + Type string `json:"type"` +} + +type ListCommonEnvCfgHistoryRes struct { + ID primitive.ObjectID `json:"id,omitempty"` + ProductName string `json:"product_name"` + CreateTime int64 `json:"create_time"` + UpdateUserName string `json:"update_user_name"` + Namespace string `json:"namespace,omitempty"` + EnvName string `json:"env_name"` + Name string `json:"name"` + Type string `json:"type"` + YamlData string `json:"yaml_data"` + SourceDetail *models.CreateFromRepo `json:"source_detail"` +} + +func GetResourceByCfgType(namespace, name string, cfgType config.CommonEnvCfgType, client client.Client, clientset *kubernetes.Clientset) (ResourceWithLabel, error) { + switch cfgType { + case config.CommonEnvCfgTypeConfigMap: + cm, _, err := getter.GetConfigMap(namespace, name, client) + return cm, err + case config.CommonEnvCfgTypeSecret: + secret, _, err := getter.GetSecret(namespace, name, client) + return secret, err + case config.CommonEnvCfgTypePvc: + pvc, _, err := getter.GetPvc(namespace, name, client) + return pvc, err + case config.CommonEnvCfgTypeIngress: + ingress, _, err := getter.GetUnstructuredIngress(namespace, name, client, clientset) + return ingress, err + } + return nil, fmt.Errorf("unrecognized env config type: %s", cfgType) +} + +func ListEnvResourceHistory(args *ListCommonEnvCfgHistoryArgs, log *zap.SugaredLogger) ([]*ListCommonEnvCfgHistoryRes, error) { + if len(args.CommonEnvCfgType) == 0 { + return nil, e.ErrListResources.AddDesc("env resource type can't be nil") + } + + product, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: args.ProjectName, + EnvName: args.EnvName, + }) + if err != nil { + return nil, e.ErrListResources.AddErr(err) + } + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), product.ClusterID) + if err != nil { + return nil, e.ErrListResources.AddErr(err) + } + clientset, err := kubeclient.GetKubeClientSet(config.HubServerAddress(), product.ClusterID) + if err != nil { + return nil, e.ErrListResources.AddErr(err) + } + + envResource, err := GetResourceByCfgType(product.Namespace, args.Name, args.CommonEnvCfgType, kubeClient, clientset) + if err != nil { + return nil, e.ErrListResources.AddErr(err) + } + if envResource == nil { + return nil, e.ErrListResources.AddDesc(fmt.Sprintf("failed to get resource %s:%s from namespace: %s", args.Name, args.CommonEnvCfgType, product.Namespace)) + } + + opts := &commonrepo.QueryEnvResourceOption{ + IsSort: true, + ProductName: args.ProjectName, + EnvName: args.EnvName, + Name: args.Name, + Type: string(args.CommonEnvCfgType), + } + res, err := commonrepo.NewEnvResourceColl().List(opts) + if err != nil { + return nil, e.ErrListResources.AddErr(err) + } + var listRes []*ListCommonEnvCfgHistoryRes + for _, resElem := range res { + listElem := &ListCommonEnvCfgHistoryRes{ + ID: resElem.ID, + ProductName: resElem.ProductName, + CreateTime: resElem.CreateTime, + UpdateUserName: resElem.UpdateUserName, + EnvName: resElem.EnvName, + Name: resElem.Name, + YamlData: resElem.YamlData, + SourceDetail: resElem.SourceDetail, + } + listRes = append(listRes, listElem) + } + return listRes, nil +} + +func ListLatestEnvResources(args *ListCommonEnvCfgHistoryArgs, log *zap.SugaredLogger) ([]*ListCommonEnvCfgHistoryRes, error) { + opts := &commonrepo.QueryEnvResourceOption{ + ProductName: args.ProjectName, + EnvName: args.EnvName, + Name: args.Name, + AutoSync: args.AutoSync, + } + res, err := commonrepo.NewEnvResourceColl().ListLatestResource(opts) + if err != nil { + return nil, e.ErrListResources.AddErr(err) + } + var listRes []*ListCommonEnvCfgHistoryRes + for _, resElem := range res { + listElem := &ListCommonEnvCfgHistoryRes{ + ProductName: resElem.ID.ProductName, + EnvName: resElem.ID.EnvName, + Type: resElem.ID.Type, + Name: resElem.ID.Name, + CreateTime: resElem.CreateTime, + } + listRes = append(listRes, listElem) + } + return listRes, nil +} + +func SyncEnvResource(args *SyncEnvResourceArg, log *zap.SugaredLogger) error { + product, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: args.ProductName, + EnvName: args.EnvName, + }) + if err != nil { + return e.ErrUpdateResource.AddErr(fmt.Errorf("failed to find product: %s:%s, err: %s", args.ProductName, args.EnvName, err)) + } + envResource, err := commonrepo.NewEnvResourceColl().Find(&commonrepo.QueryEnvResourceOption{ + ProductName: args.ProductName, + EnvName: args.EnvName, + Name: args.Name, + Type: args.Type, + }) + if err != nil { + return e.ErrUpdateResource.AddErr(fmt.Errorf("failed to find env resource: %s:%s of env: %s:%s, err: %s", args.Type, args.Name, args.ProductName, args.EnvName, err)) + } + + if !envResource.AutoSync { + return nil + } + + if envResource.SourceDetail == nil || envResource.SourceDetail.GitRepoConfig == nil { + return e.ErrUpdateResource.AddErr(fmt.Errorf("failed to parse gitops config for resource: %s:%s of env: %s:%s", args.Type, args.Name, args.ProductName, args.EnvName)) + } + + repoConfig := envResource.SourceDetail.GitRepoConfig + sourceYaml, err := fsservice.DownloadFileFromSource(&fsservice.DownloadFromSourceArgs{ + CodehostID: repoConfig.CodehostID, + Namespace: repoConfig.GetNamespace(), + Owner: repoConfig.Owner, + Repo: repoConfig.Repo, + Path: envResource.SourceDetail.LoadPath, + Branch: repoConfig.Branch, + }) + if err != nil { + return e.ErrUpdateResource.AddErr(err) + } + + js, err := yaml.YAMLToJSON(sourceYaml) + if err != nil { + return e.ErrUpdateResource.AddErr(err) + } + + resName, yamlData, err := getResourceYamlAndName(js, product.Namespace, args.ProductName, config.CommonEnvCfgType(envResource.Type)) + if err != nil { + return e.ErrUpdateResource.AddErr(err) + } + + // asset resource name be the same + if resName != envResource.Name { + return e.ErrUpdateResource.AddDesc(fmt.Sprintf("resource name not match, expect: %s while parsed: %s", envResource.Name, resName)) + } + + equal, err := yamlutil.Equal(yamlData, envResource.YamlData) + if err != nil { + return e.ErrUpdateResource.AddErr(err) + } + + if equal { + return nil + } + + err = UpdateCommonEnvCfg(&models.CreateUpdateCommonEnvCfgArgs{ + EnvName: args.EnvName, + ProductName: args.ProductName, + Name: args.Name, + YamlData: yamlData, + RestartAssociatedSvc: false, + CommonEnvCfgType: config.CommonEnvCfgType(envResource.Type), + AutoSync: envResource.AutoSync, + LatestEnvResource: envResource, + }, "cron", true, log) + if err != nil { + return e.ErrUpdateResource.AddErr(err) + } + return nil +} + +func restartPod(name, productName, envName, namespace string, commonEnvCfgType config.CommonEnvCfgType, clientset *kubernetes.Clientset, kubcli client.Client) error { + opts := &commonrepo.ListEnvSvcDependOption{ + ProductName: productName, + EnvName: envName, + } + envSvcProducts, err := commonrepo.NewEnvSvcDependColl().List(opts) + if err != nil { + return err + } + tplProduct, err := template.NewProductColl().Find(productName) + if err != nil { + return err + } + + for _, esp := range envSvcProducts { + switch commonEnvCfgType { + case config.CommonEnvCfgTypeConfigMap: + if !checkExistInList(name, esp.ConfigMaps) { + continue + } + case config.CommonEnvCfgTypeSecret: + if !checkExistInList(name, esp.Secrets) { + continue + } + case config.CommonEnvCfgTypePvc: + if !checkExistInList(name, esp.Pvcs) { + continue + } + default: + return fmt.Errorf("unsupport type %s to restart service", commonEnvCfgType) + } + + restartArgs := &SvcOptArgs{ + EnvName: envName, + ProductName: productName, + ServiceName: esp.ServiceName, + } + if tplProduct.ProductFeature.DeployType == "k8s" { + if err := restartK8sPod(restartArgs, namespace, clientset); err != nil { + log.Error(err) + return err + } + continue + } + + if err := restartPodHelmByWorkload(restartArgs, namespace, clientset, kubcli); err != nil { + log.Error(err) + return err + } + } + return nil +} + +func restartK8sPod(args *SvcOptArgs, ns string, clientset *kubernetes.Clientset) error { + selector := labels.Set{setting.ProductLabel: args.ProductName, setting.ServiceLabel: args.ServiceName}.AsSelector() + log.Infof("deleting pod from %s where %s", ns, selector) + return updater.DeletePods(ns, selector, clientset) +} + +func restartPodHelmByWorkload(args *SvcOptArgs, ns string, clientset *kubernetes.Clientset, kucli client.Client) error { + deployment, found, err := getter.GetDeployment(ns, args.ServiceName, kucli) + if err != nil { + return err + } + if found { + selector := labels.Set(deployment.GetLabels()).AsSelector() + log.Infof("deleting deploy %s pod from %s where %s", args.ServiceName, ns, selector) + return updater.DeletePods(ns, selector, clientset) + } + + sts, found, err := getter.GetStatefulSet(ns, args.ServiceName, kucli) + if err != nil { + return err + } + if found { + selector := labels.Set(sts.GetLabels()).AsSelector() + log.Infof("deleting sts %s pod from %s where %s", args.ServiceName, ns, selector) + return updater.DeletePods(ns, selector, clientset) + } + + return nil +} + +func checkExistInList(target string, arrList []string) bool { + for _, arr := range arrList { + if target == arr { + return true + } + } + return false +} diff --git a/pkg/microservice/aslan/core/environment/service/common_env_cfg_informer.go b/pkg/microservice/aslan/core/environment/service/common_env_cfg_informer.go new file mode 100644 index 0000000000000000000000000000000000000000..040b5149600290b38c9d1c9e13fc13a42d05f1fd --- /dev/null +++ b/pkg/microservice/aslan/core/environment/service/common_env_cfg_informer.go @@ -0,0 +1,381 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "context" + "sync" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" + "github.com/koderover/zadig/pkg/tool/log" +) + +var ClusterInformersMap sync.Map +var ClusterStopChanMap sync.Map + +func NewClusterInformerFactory(clusterID string, cls *kubernetes.Clientset) (informers.SharedInformerFactory, error) { + if clusterID == "" { + clusterID = setting.LocalClusterID + } + if informer, ok := ClusterInformersMap.Load(clusterID); ok { + return informer.(informers.SharedInformerFactory), nil + } + + informerFactory := informers.NewSharedInformerFactoryWithOptions(cls, time.Minute) + informerFactory.Apps().V1().Deployments().Lister() + informerFactory.Apps().V1().Deployments().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + go onDeploymentAddAndUpdate(obj) + }, + UpdateFunc: func(oldObj interface{}, newObj interface{}) { + newDeploy := newObj.(*appsv1.Deployment) + oldDeploy := oldObj.(*appsv1.Deployment) + if newDeploy.ResourceVersion == oldDeploy.ResourceVersion { + return + } + go onDeploymentAddAndUpdate(newObj) + }, + DeleteFunc: onDeploymentDelete, + }) + + informerFactory.Apps().V1().StatefulSets().Lister() + informerFactory.Apps().V1().StatefulSets().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + go onStatefulSetAddAndUpdate(obj) + }, + UpdateFunc: func(oldObj interface{}, newObj interface{}) { + newDeploy := newObj.(*appsv1.StatefulSet) + oldDeploy := oldObj.(*appsv1.StatefulSet) + if newDeploy.ResourceVersion == oldDeploy.ResourceVersion { + return + } + go onStatefulSetAddAndUpdate(newObj) + }, + DeleteFunc: onStatefulSetDelete, + }) + + stopchan := make(chan struct{}) + informerFactory.Start(stopchan) + informerFactory.WaitForCacheSync(make(chan struct{})) + if _, ok := ClusterInformersMap.Load(clusterID); ok { + if stopchan, ok := ClusterStopChanMap.Load(clusterID); ok { + close(stopchan.(chan struct{})) + ClusterStopChanMap.Delete(clusterID) + } + } + ClusterInformersMap.Store(clusterID, informerFactory) + ClusterStopChanMap.Store(clusterID, stopchan) + return informerFactory, nil +} + +func DeleteClusterInformer(clusterID string) { + if _, ok := ClusterInformersMap.Load(clusterID); ok { + if stopchan, ok := ClusterStopChanMap.Load(clusterID); ok { + close(stopchan.(chan struct{})) + ClusterStopChanMap.Delete(clusterID) + } + } + ClusterInformersMap.Delete(clusterID) +} + +func onDeploymentAddAndUpdate(obj interface{}) { + deploy := obj.(*appsv1.Deployment) + labels := deploy.GetLabels() + serviceName := labels[setting.ServiceLabel] + product, continueFlag := GetProductAndFilterNs(deploy.Namespace, deploy.Name, serviceName) + if !continueFlag { + return + } + + configMaps, secrets, pvcs := VisitDeployment(deploy) + envSvcDepend := &models.EnvSvcDepend{ + ProductName: product.ProductName, + EnvName: product.EnvName, + Namespace: deploy.Namespace, + ServiceName: serviceName, + ServiceModule: deploy.Name, + WorkloadType: setting.Deployment, + ConfigMaps: configMaps.List(), + Pvcs: pvcs.List(), + Secrets: secrets.List(), + } + if err := commonrepo.NewEnvSvcDependColl().CreateOrUpdate(envSvcDepend); err != nil { + log.Error("onDeploymentAdd EnvSvcDepend CreateOrUpdate error:", err) + } +} + +func onDeploymentDelete(obj interface{}) { + deploy := obj.(*appsv1.Deployment) + labels := deploy.GetLabels() + serviceName := labels[setting.ServiceLabel] + product, continueFlag := GetProductAndFilterNs(deploy.Namespace, deploy.Name, serviceName) + if !continueFlag { + return + } + opts := &commonrepo.DeleteEnvSvcDependOption{ + ProductName: product.ProductName, + EnvName: product.EnvName, + ServiceName: serviceName, + ServiceModule: deploy.Name, + } + if err := commonrepo.NewEnvSvcDependColl().Delete(opts); err != nil { + log.Error("onDeploymentDelete EnvSvcDepend Delete error:", err) + } +} + +func VisitDeployment(deployment *appsv1.Deployment) (sets.String, sets.String, sets.String) { + cfgSets := sets.NewString() + secretSets := sets.NewString() + pvcSets := sets.NewString() + for _, initCon := range deployment.Spec.Template.Spec.InitContainers { + cfg, sec := visitContainerConfigmapAndSecretNames(initCon) + cfgSets = cfgSets.Union(cfg) + secretSets = secretSets.Union(sec) + } + for _, con := range deployment.Spec.Template.Spec.Containers { + cfg, sec := visitContainerConfigmapAndSecretNames(con) + cfgSets = cfgSets.Union(cfg) + secretSets = secretSets.Union(sec) + } + for _, ephemeralCon := range deployment.Spec.Template.Spec.EphemeralContainers { + cfg, sec := visitContainerConfigmapAndSecretNames(corev1.Container(ephemeralCon.DeepCopy().EphemeralContainerCommon)) + cfgSets = cfgSets.Union(cfg) + secretSets = secretSets.Union(sec) + } + for _, volume := range deployment.Spec.Template.Spec.Volumes { + cfg, sec, pvc := visitVolumeConfigmapAndSecretAndPvcNames(volume) + cfgSets = cfgSets.Union(cfg) + secretSets = secretSets.Union(sec) + pvcSets = pvcSets.Union(pvc) + } + for _, sec := range deployment.Spec.Template.Spec.ImagePullSecrets { + secretSets.Insert(sec.Name) + } + return cfgSets, secretSets, pvcSets +} + +func onStatefulSetAddAndUpdate(obj interface{}) { + sts := obj.(*appsv1.StatefulSet) + labels := sts.GetLabels() + serviceName := labels[setting.ServiceLabel] + product, continueFlag := GetProductAndFilterNs(sts.Namespace, sts.Name, serviceName) + if !continueFlag { + return + } + + configMaps, secrets, pvcs := VisitStatefulSet(sts) + envSvcDepend := &models.EnvSvcDepend{ + ProductName: product.ProductName, + EnvName: product.EnvName, + Namespace: sts.Namespace, + ServiceName: serviceName, + ServiceModule: sts.Name, + WorkloadType: setting.StatefulSet, + ConfigMaps: configMaps.List(), + Pvcs: pvcs.List(), + Secrets: secrets.List(), + } + + if err := commonrepo.NewEnvSvcDependColl().CreateOrUpdate(envSvcDepend); err != nil { + log.Error("onStatefulSetAdd EnvSvcDepend CreateOrUpdate error:", err) + } +} + +func onStatefulSetDelete(obj interface{}) { + sts := obj.(*appsv1.StatefulSet) + labels := sts.GetLabels() + serviceName := labels[setting.ServiceLabel] + product, continueFlag := GetProductAndFilterNs(sts.Namespace, sts.Name, serviceName) + if !continueFlag { + return + } + opts := &commonrepo.DeleteEnvSvcDependOption{ + ProductName: product.ProductName, + EnvName: product.EnvName, + ServiceName: serviceName, + ServiceModule: sts.Name, + } + if err := commonrepo.NewEnvSvcDependColl().Delete(opts); err != nil { + log.Error("onStatefulSetDelete EnvSvcDepend Delete error:", err) + } +} + +func VisitStatefulSet(sts *appsv1.StatefulSet) (sets.String, sets.String, sets.String) { + cfgSets := sets.NewString() + secretSets := sets.NewString() + pvcSets := sets.NewString() + for _, initCon := range sts.Spec.Template.Spec.InitContainers { + cfg, sec := visitContainerConfigmapAndSecretNames(initCon) + cfgSets = cfgSets.Union(cfg) + secretSets = secretSets.Union(sec) + } + for _, con := range sts.Spec.Template.Spec.Containers { + cfg, sec := visitContainerConfigmapAndSecretNames(con) + cfgSets = cfgSets.Union(cfg) + secretSets = secretSets.Union(sec) + } + for _, ephemeralCon := range sts.Spec.Template.Spec.EphemeralContainers { + cfg, sec := visitContainerConfigmapAndSecretNames(corev1.Container(ephemeralCon.DeepCopy().EphemeralContainerCommon)) + cfgSets = cfgSets.Union(cfg) + secretSets = secretSets.Union(sec) + } + for _, volume := range sts.Spec.Template.Spec.Volumes { + cfg, sec, pvc := visitVolumeConfigmapAndSecretAndPvcNames(volume) + cfgSets = cfgSets.Union(cfg) + secretSets = secretSets.Union(sec) + pvcSets = pvcSets.Union(pvc) + } + for _, sec := range sts.Spec.Template.Spec.ImagePullSecrets { + secretSets.Insert(sec.Name) + } + return cfgSets, secretSets, pvcSets +} + +func GetProductAndFilterNs(namespace, workloadName, svcName string) (*models.Product, bool) { + products, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{Namespace: namespace}) + if err != nil { + if !commonrepo.IsErrNoDocuments(err) { + log.Error("GetProductAndFilterNs err:", err) + } + return nil, false + } + + for _, product := range products { + if !product.IsExisted { + clientset, err := kubeclient.GetKubeClientSet(config.HubServerAddress(), product.ClusterID) + if err != nil { + log.Errorf("failed to create kubernetes clientset for clusterID: %s, the error is: %s", product.ClusterID, err) + return nil, false + } + ns, err := clientset.CoreV1().Namespaces().Get(context.Background(), namespace, metav1.GetOptions{}) + if err != nil { + return nil, false + } + labels := ns.GetLabels() + if val, ok := labels[setting.EnvCreatedBy]; !ok || val != setting.EnvCreator { + return nil, false + } + } + for _, psvc := range product.GetServiceMap() { + if psvc.Type == setting.K8SDeployType && (psvc.ServiceName == workloadName || psvc.ServiceName == svcName) { + return product, true + } + } + } + return nil, false +} + +func visitContainerConfigmapAndSecretNames(container corev1.Container) (sets.String, sets.String) { + cfgSets := sets.NewString() + secretSets := sets.NewString() + for _, env := range container.EnvFrom { + if env.ConfigMapRef != nil { + cfgSets.Insert(env.ConfigMapRef.Name) + } + if env.SecretRef != nil { + secretSets.Insert(env.SecretRef.Name) + } + } + for _, envVar := range container.Env { + if envVar.ValueFrom != nil && envVar.ValueFrom.ConfigMapKeyRef != nil { + cfgSets.Insert(envVar.ValueFrom.ConfigMapKeyRef.Name) + } + if envVar.ValueFrom != nil && envVar.ValueFrom.SecretKeyRef != nil { + secretSets.Insert(envVar.ValueFrom.SecretKeyRef.Name) + } + } + return cfgSets, secretSets +} + +func visitVolumeConfigmapAndSecretAndPvcNames(volume corev1.Volume) (sets.String, sets.String, sets.String) { + cfgSets := sets.NewString() + secretSets := sets.NewString() + pvcSets := sets.NewString() + source := &volume.VolumeSource + switch { + case source.Projected != nil: + for j := range source.Projected.Sources { + if source.Projected.Sources[j].ConfigMap != nil { + cfgSets.Insert(source.Projected.Sources[j].ConfigMap.Name) + } + if source.Projected.Sources[j].Secret != nil { + secretSets.Insert(source.Projected.Sources[j].Secret.Name) + } + } + case source.ConfigMap != nil: + cfgSets.Insert(source.ConfigMap.Name) + case source.Secret != nil: + secretSets.Insert(source.Secret.SecretName) + case source.PersistentVolumeClaim != nil: + pvcSets.Insert(source.PersistentVolumeClaim.ClaimName) + } + return cfgSets, secretSets, pvcSets +} + +func StartClusterInformer() { + for { + k8sClusters, err := mongodb.NewK8SClusterColl().List(nil) + if err != nil && !mongodb.IsErrNoDocuments(err) { + log.Error("StartClusterInformer list cluster error:%s", err) + return + } + + for _, cluster := range k8sClusters { + if cluster.Status != setting.Normal { + DeleteClusterInformer(cluster.ID.Hex()) + continue + } + clientset, err := kubeclient.GetKubeClientSet(config.HubServerAddress(), cluster.ID.Hex()) + if err != nil { + log.Errorf("failed to create kubernetes clientset for clusterID: %s, the error is: %s", cluster.ID.Hex(), err) + continue + } + + _, err = NewClusterInformerFactory(cluster.ID.Hex(), clientset) + if err != nil { + log.Errorf("failed to NewClusterInformerFactory clusterID: %s, the error is: %s", cluster.ID.Hex(), err) + } + } + + deleteClustInf := func(key, value interface{}) bool { + for _, cluster := range k8sClusters { + if key.(string) == cluster.ID.Hex() { + return true + } + } + DeleteClusterInformer(key.(string)) + return true + } + ClusterInformersMap.Range(deleteClustInf) + + time.Sleep(5 * time.Second) + } +} diff --git a/pkg/microservice/aslan/core/environment/service/configmap.go b/pkg/microservice/aslan/core/environment/service/configmap.go index 045d23381c90d0b1cedd7dca2223279ad446b8af..18ecb6bfec23adb3740919a8a17880007697aad0 100644 --- a/pkg/microservice/aslan/core/environment/service/configmap.go +++ b/pkg/microservice/aslan/core/environment/service/configmap.go @@ -17,27 +17,32 @@ limitations under the License. package service import ( + "encoding/json" "fmt" "sort" - "strconv" "strings" + "sync" "time" + "go.mongodb.org/mongo-driver/mongo" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + commontpl "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/kube/wrapper" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/kube/getter" "github.com/koderover/zadig/pkg/tool/kube/updater" - "github.com/koderover/zadig/pkg/tool/kube/util" ) type ListConfigMapArgs struct { @@ -55,42 +60,28 @@ type RollBackConfigMapArgs struct { } type UpdateConfigMapArgs struct { - EnvName string `json:"env_name"` - ProductName string `json:"product_name"` - ServiceName string `json:"service_name"` - ConfigName string `json:"config_name"` - Data map[string]string `json:"data"` + EnvName string `json:"env_name"` + ProductName string `json:"product_name"` + ServiceName string `json:"service_name"` + ConfigName string `json:"config_name"` + YamlData string `json:"yaml_data"` + RestartAssociatedSvc bool `json:"restart_associated_svc"` + Services []string `json:"services"` } -type configMap struct { - Name string `json:"name"` - Data map[string]string `json:"data"` - ModifiedBy string `json:"modifiedBy"` - CreationTimestamp string `json:"creationTimestamp"` - LastUpdateTime string `json:"lastUpdateTime"` - lastUpdateTime time.Time +type ListConfigMapRes struct { + *ResourceResponseBase + CmName string `json:"cm_name"` + Immutable bool `json:"immutable"` + CmData map[string]string `json:"cm_data"` } -type configMapWithHistory struct { - Current *configMap `json:"current"` - HistoricalRevisions []*configMap `json:"historicalRevisions"` -} - -func generateConfigMapResponse(cm *corev1.ConfigMap) *configMap { - icm := wrapper.ConfigMap(cm) - u, _ := icm.LastUpdateTime() - return &configMap{ - Name: icm.Name, - Data: icm.Data, - ModifiedBy: icm.ModifiedBy(), - CreationTimestamp: util.FormatTime(icm.CreationTimestamp.Time), - LastUpdateTime: util.FormatTime(u), - lastUpdateTime: u, +func ListConfigMaps(args *ListConfigMapArgs, log *zap.SugaredLogger) ([]*ListConfigMapRes, error) { + selector := labels.Set{}.AsSelector() + // Note. when listing configs from [配置管理] on workload page, service name will be passed as query condition + if args.ServiceName != "" { + selector = labels.Set{setting.ProductLabel: args.ProductName, setting.ServiceLabel: args.ServiceName}.AsSelector() } -} - -func ListConfigMaps(args *ListConfigMapArgs, log *zap.SugaredLogger) ([]*configMapWithHistory, error) { - selector := labels.Set{setting.ProductLabel: args.ProductName, setting.ServiceLabel: args.ServiceName}.AsSelector() product, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ Name: args.ProductName, @@ -99,7 +90,7 @@ func ListConfigMaps(args *ListConfigMapArgs, log *zap.SugaredLogger) ([]*configM if err != nil { return nil, e.ErrListConfigMaps.AddErr(err) } - kubeClient, err := kube.GetKubeClient(product.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), product.ClusterID) if err != nil { return nil, e.ErrListConfigMaps.AddErr(err) } @@ -110,43 +101,98 @@ func ListConfigMaps(args *ListConfigMapArgs, log *zap.SugaredLogger) ([]*configM return nil, e.ErrListConfigMaps.AddDesc(err.Error()) } - // 为了兼容老数据，需要找到当前使用的cm，即labels中没有config-backup=true的cm - // 当前使用的cm需要放在数组的首位，备份的cm则按照创建时间倒序 - cmWithHistory := make(map[string][]*configMap) - cmWithName := make(map[string]*configMap) - for _, cm := range cms { - icm := wrapper.ConfigMap(cm) - res := generateConfigMapResponse(cm) - if icm.Active() { - cmWithName[res.Name] = res - } else { - owner := icm.Owner() - if owner == "" { - continue + envSvcDepends, err := commonrepo.NewEnvSvcDependColl().List(&commonrepo.ListEnvSvcDependOption{ProductName: args.ProductName, EnvName: args.EnvName}) + if err != nil && commonrepo.IsErrNoDocuments(err) { + return nil, e.ErrListResources.AddDesc(err.Error()) + } + + var res []*ListConfigMapRes + var mutex sync.Mutex + var wg sync.WaitGroup + cmProcess := func(cm *corev1.ConfigMap) error { + for labelKey, labelVal := range cm.GetLabels() { + if labelKey == setting.ConfigBackupLabel && labelVal == setting.LabelValueTrue { + return nil } - cmWithHistory[owner] = append(cmWithHistory[owner], res) } - } + cm.SetManagedFields(nil) + cm.SetResourceVersion("") + if cm.APIVersion == "" { + cm.APIVersion = "v1" + } + if cm.Kind == "" { + cm.Kind = "ConfigMap" + } + yamlData, err := yaml.Marshal(cm) + if err != nil { + return err + } - res := make([]*configMapWithHistory, 0, len(cmWithName)) - for name, cm := range cmWithName { - history := cmWithHistory[name] - if len(history) > 1 { - sort.SliceStable(history, func(i, j int) bool { - return history[i].lastUpdateTime.After(history[j].lastUpdateTime) - }) + var tempSvcs []string + LB: + for _, svcDepend := range envSvcDepends { + for _, ccm := range svcDepend.ConfigMaps { + if ccm == cm.Name { + tempSvcs = append(tempSvcs, svcDepend.ServiceModule) + continue LB + } + } } - res = append(res, &configMapWithHistory{ - Current: cm, - HistoricalRevisions: history, - }) + immutable := false + if cm.Immutable != nil { + immutable = *cm.Immutable + } + resElem := &ListConfigMapRes{ + CmName: cm.Name, + Immutable: immutable, + CmData: cm.Data, + ResourceResponseBase: &ResourceResponseBase{ + Name: cm.Name, + Type: config.CommonEnvCfgTypeConfigMap, + EnvName: args.EnvName, + ProjectName: args.ProductName, + YamlData: string(yamlData), + Services: tempSvcs, + CreateTime: cm.GetCreationTimestamp().Time, + }, + } + resElem.setSourceDetailData(cm) + + mutex.Lock() + res = append(res, resElem) + mutex.Unlock() + return nil + } + + for _, cmElem := range cms { + wg.Add(1) + go func(cm *corev1.ConfigMap) { + defer wg.Done() + err := cmProcess(cm) + if err != nil { + log.Errorf("ListConfigMaps ns:%s name:%s cmProcess err:%s", cm.Namespace, cm.Name, err) + } + }(cmElem) } + wg.Wait() + sort.SliceStable(res, func(i, j int) bool { + return res[i].CmName < res[j].CmName + }) return res, nil } -func UpdateConfigMap(envName string, args *UpdateConfigMapArgs, userName string, userID int, log *zap.SugaredLogger) error { +func UpdateConfigMap(args *models.CreateUpdateCommonEnvCfgArgs, userName string, log *zap.SugaredLogger) error { + js, err := yaml.YAMLToJSON([]byte(args.YamlData)) + cm := &corev1.ConfigMap{} + err = json.Unmarshal(js, cm) + if err != nil { + return e.ErrUpdateConfigMap.AddErr(err) + } + if cm.Name != args.Name { + return e.ErrUpdateConfigMap.AddDesc("configMap Yaml Name is incorrect") + } product, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ Name: args.ProductName, EnvName: args.EnvName, @@ -154,76 +200,69 @@ func UpdateConfigMap(envName string, args *UpdateConfigMapArgs, userName string, if err != nil { return e.ErrUpdateConfigMap.AddErr(err) } - kubeClient, err := kube.GetKubeClient(product.ClusterID) - if err != nil { - return e.ErrUpdateConfigMap.AddErr(err) - } namespace := product.Namespace - cfg, found, err := getter.GetConfigMap(namespace, args.ConfigName, kubeClient) - if err != nil { - log.Error(err) - return e.ErrGetConfigMap.AddDesc(err.Error()) - } else if !found { - return e.ErrGetConfigMap.AddDesc("configMap not found") - } - - if err := archiveConfigMap(namespace, cfg, kubeClient, log); err != nil { - return err - } - - // 将configMap中的变量进行渲染 - renderSet, err := commonservice.GetRenderSet(namespace, 0, log) + renderSet, err := commonservice.GetRenderSet(namespace, 0, false, product.EnvName, log) if err != nil { log.Errorf("Failed to find render set for product template %s, err: %v", product.ProductName, err) return err } - - // 渲染变量 - for key, value := range args.Data { + for key, value := range cm.Data { for _, kv := range renderSet.KVs { value = strings.Replace(value, kv.Alias, kv.Value, -1) } value = kube.ParseSysKeys(product.Namespace, product.EnvName, product.ProductName, args.ServiceName, value) - args.Data[key] = value + cm.Data[key] = value } - cfg.Data = args.Data - // 记录修改configmap的用户 - cfg.Labels[setting.UpdateBy] = kube.MakeSafeLabelValue(userName) - cfg.Labels[setting.UpdateByID] = fmt.Sprintf("%d", userID) - cfg.Labels[setting.UpdateTime] = time.Now().Format("20060102150405") - cfg.Labels[setting.DirtyLabel] = setting.LabelValueTrue + clientset, err := kubeclient.GetKubeClientSet(config.HubServerAddress(), product.ClusterID) + if err != nil { + log.Errorf("failed to create kubernetes clientset for clusterID: %s, the error is: %s", product.ClusterID, err) + return e.ErrUpdateConfigMap.AddErr(err) + } - as := cfg.GetAnnotations() - if as == nil { - as = make(map[string]string) + yamlData, err := ensureLabelAndNs(cm, product.Namespace, args.ProductName) + if err != nil { + return e.ErrUpdateResource.AddErr(err) } - as[setting.ModifiedByAnnotation] = userName - as[setting.EditorIDAnnotation] = strconv.Itoa(userID) - as[setting.LastUpdateTimeAnnotation] = util.FormatTime(time.Now()) - cfg.SetAnnotations(as) - if err := updater.UpdateConfigMap(cfg, kubeClient); err != nil { + if err := updater.UpdateConfigMap(namespace, cm, clientset); err != nil { log.Error(err) return e.ErrUpdateConfigMap.AddDesc(err.Error()) } - - restartArgs := &SvcOptArgs{ - EnvName: envName, - ProductName: args.ProductName, - ServiceName: args.ServiceName, + envCM := &models.EnvResource{ + ProductName: args.ProductName, + UpdateUserName: userName, + EnvName: args.EnvName, + Namespace: product.Namespace, + Name: cm.Name, + YamlData: yamlData, + Type: string(config.CommonEnvCfgTypeConfigMap), + SourceDetail: args.SourceDetail, + AutoSync: args.AutoSync, } - - if err := restartPod(restartArgs, namespace, kubeClient, log); err != nil { - log.Error(err) + if err := commonrepo.NewEnvResourceColl().Create(envCM); err != nil { + return e.ErrUpdateConfigMap.AddDesc(err.Error()) + } + tplProduct, err := commontpl.NewProductColl().Find(args.ProductName) + if err != nil { + return e.ErrUpdateConfigMap.AddDesc(err.Error()) + } + //TODO: helm not support restart service + if !args.RestartAssociatedSvc || tplProduct.ProductFeature.DeployType != setting.K8SDeployType { + return nil + } + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), product.ClusterID) + if err != nil { + return e.ErrUpdateConfigMap.AddErr(err) + } + if err := restartPod(cm.Name, args.ProductName, args.EnvName, namespace, config.CommonEnvCfgTypeConfigMap, clientset, kubeClient); err != nil { return e.ErrRestartService.AddDesc(err.Error()) } - return nil } -func RollBackConfigMap(envName string, args *RollBackConfigMapArgs, userName string, userID int, log *zap.SugaredLogger) error { +func RollBackConfigMap(envName string, args *RollBackConfigMapArgs, userName, userID string, log *zap.SugaredLogger) error { product, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ Name: args.ProductName, EnvName: args.EnvName, @@ -231,11 +270,17 @@ func RollBackConfigMap(envName string, args *RollBackConfigMapArgs, userName str if err != nil { return e.ErrUpdateConfigMap.AddErr(err) } - kubeClient, err := kube.GetKubeClient(product.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), product.ClusterID) if err != nil { return e.ErrUpdateConfigMap.AddErr(err) } + clientset, err := kubeclient.GetKubeClientSet(config.HubServerAddress(), product.ClusterID) + if err != nil { + log.Errorf("failed to create kubernetes clientset for clusterID: %s, the error is: %s", product.ClusterID, err) + return e.ErrUpdateConfigMap.AddErr(err) + } + namespace := product.Namespace srcCfg, found, err := getter.GetConfigMap(namespace, args.SrcConfigName, kubeClient) if err != nil { @@ -260,13 +305,13 @@ func RollBackConfigMap(envName string, args *RollBackConfigMapArgs, userName str destinSrc.Data = srcCfg.Data destinSrc.Labels[setting.UpdateBy] = kube.MakeSafeLabelValue(userName) - destinSrc.Labels[setting.UpdateByID] = fmt.Sprintf("%d", userID) + destinSrc.Labels[setting.UpdateByID] = userID destinSrc.Labels[setting.UpdateTime] = time.Now().Format("20060102150405") // 回滚时显示回滚版本的时间 if updateTime, ok := srcCfg.Labels[setting.UpdateTime]; ok { destinSrc.Labels[setting.UpdateTime] = updateTime } - if err := updater.UpdateConfigMap(destinSrc, kubeClient); err != nil { + if err := updater.UpdateConfigMap(namespace, destinSrc, clientset); err != nil { log.Error(err) return e.ErrUpdateConfigMap.AddDesc(err.Error()) } @@ -277,7 +322,7 @@ func RollBackConfigMap(envName string, args *RollBackConfigMapArgs, userName str ServiceName: args.ServiceName, } - if err := restartPod(restartArgs, namespace, kubeClient, log); err != nil { + if err := restartK8sPod(restartArgs, namespace, clientset); err != nil { log.Error(err) return e.ErrRestartService.AddDesc(err.Error()) } @@ -345,9 +390,81 @@ func cleanArchiveConfigMap(namespace string, ls map[string]string, kubeClient cl } } -func restartPod(args *SvcOptArgs, ns string, kubeClient client.Client, log *zap.SugaredLogger) error { - selector := labels.Set{setting.ProductLabel: args.ProductName, setting.ServiceLabel: args.ServiceName}.AsSelector() +type MigrateHistoryConfigMapsRes struct { +} - log.Infof("deleting pod from %s where %s", ns, selector) - return updater.DeletePods(ns, selector, kubeClient) +func MigrateHistoryConfigMaps(envName, productName string, log *zap.SugaredLogger) ([]*models.EnvResource, error) { + + res := make([]*models.EnvResource, 0) + products := make([]*models.Product, 0) + var err error + if productName != "" { + products, err = commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{ + Name: productName, + EnvName: envName, + ExcludeStatus: []string{setting.ProductStatusDeleting, setting.ProductStatusCreating}, + }) + } else { + products, err = commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{ + ExcludeStatus: []string{setting.ProductStatusDeleting, setting.ProductStatusCreating}, + }) + } + if err != nil && err != mongo.ErrNoDocuments { + return nil, err + } + + for _, product := range products { + if product.IsExisted { + continue + } + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), product.ClusterID) + if err != nil { + return nil, e.ErrListResources.AddErr(err) + } + cms, err := getter.ListConfigMaps(product.Namespace, labels.Set{setting.ProductLabel: product.ProductName, setting.ConfigBackupLabel: setting.LabelValueTrue}.AsSelector(), kubeClient) + if err != nil { + return nil, e.ErrListResources.AddErr(err) + } + + for _, cm := range cms { + cmNames := strings.Split(cm.Name, "-bak-") + cmName := cmNames[0] + envResource := &models.EnvResource{ + ProductName: product.ProductName, + EnvName: product.EnvName, + Namespace: product.Namespace, + Name: cmName, + Type: string(config.CommonEnvCfgTypeConfigMap), + } + cmLables := cm.GetLabels() + if _, ok := cmLables[setting.UpdateTime]; ok { + tm, err := time.Parse("20060102150405", cmLables[setting.UpdateTime]) + if err != nil { + return nil, e.ErrListResources.AddErr(err) + } + envResource.CreateTime = tm.Unix() + } + + delete(cmLables, setting.UpdateTime) + delete(cmLables, setting.ConfigBackupLabel) + cm.SetLabels(cmLables) + cm.SetManagedFields(nil) + cm.SetResourceVersion("") + cm.SetAnnotations(make(map[string]string)) + cm.SetName(cmName) + + yamlData, err := ensureLabelAndNs(cm, product.Namespace, productName) + if err != nil { + return nil, e.ErrListResources.AddDesc(err.Error()) + } + + envResource.YamlData = yamlData + err = commonrepo.NewEnvResourceColl().Create(envResource) + if err != nil { + return nil, e.ErrListResources.AddErr(err) + } + res = append(res, envResource) + } + } + return res, nil } diff --git a/pkg/microservice/aslan/core/environment/service/debug.go b/pkg/microservice/aslan/core/environment/service/debug.go new file mode 100644 index 0000000000000000000000000000000000000000..9bb10ebed49a7cba64bf0ec66ad6134bf1ab5258 --- /dev/null +++ b/pkg/microservice/aslan/core/environment/service/debug.go @@ -0,0 +1,182 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/strategicpatch" + "k8s.io/apimachinery/pkg/version" + "k8s.io/client-go/discovery" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" +) + +const ZadigDebugContainerName = "zadig-debug" +const K8sBetaVersionForEphemeralContainer = "v1.23" + +func PatchDebugContainer(ctx context.Context, projectName, envName, podName, debugImage string) error { + prod, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: projectName, + EnvName: envName, + }) + if err != nil { + return fmt.Errorf("failed to query env %q in project %q: %s", envName, projectName, err) + } + + clusterID := prod.ClusterID + ns := prod.Namespace + + kclient, err := kubeclient.GetKubeClient(config.HubServerAddress(), clusterID) + if err != nil { + return fmt.Errorf("failed to get kube client: %s", err) + } + + clientset, err := kubeclient.GetKubeClientSet(config.HubServerAddress(), clusterID) + if err != nil { + return fmt.Errorf("failed to get kube clientset: %s", err) + } + + restConfig, err := kubeclient.GetRESTConfig(config.HubServerAddress(), clusterID) + if err != nil { + return fmt.Errorf("failed to get rest config: %s", err) + } + + discoveryClient, err := discovery.NewDiscoveryClientForConfig(restConfig) + if err != nil { + return fmt.Errorf("failed to get discovery client: %s", err) + } + + pod := &corev1.Pod{} + err = kclient.Get(ctx, client.ObjectKey{ + Name: podName, + Namespace: ns, + }, pod) + if err != nil { + return fmt.Errorf("failed to get pod %q in ns %q: %s", podName, ns, err) + } + + k8sVersion, err := checkK8sVersion(discoveryClient) + if err != nil { + return fmt.Errorf("failed to check K8s version: %s", err) + } + + debugContainer := genDebugContainer(debugImage) + if version.CompareKubeAwareVersionStrings(K8sBetaVersionForEphemeralContainer, k8sVersion) < 0 { + _, _, err = debugByEphemeralContainerLegacy(ctx, clientset.CoreV1(), pod, debugContainer) + } else { + _, _, err = debugByEphemeralContainer(ctx, clientset.CoreV1(), pod, debugContainer) + } + + return err +} + +func checkK8sVersion(client *discovery.DiscoveryClient) (string, error) { + serverInfo, err := client.ServerVersion() + if err != nil { + return "", err + } + + // Examples: v1.23.3, v1.20.6-tke.16 + items := strings.Split(serverInfo.GitVersion, ".") + if len(items) < 2 { + return "", fmt.Errorf("invalid server version format %q", serverInfo.GitVersion) + } + + return fmt.Sprintf("%s.%s", items[0], items[1]), nil +} + +func genDebugContainer(imageName string) *corev1.EphemeralContainer { + return &corev1.EphemeralContainer{ + EphemeralContainerCommon: corev1.EphemeralContainerCommon{ + Name: ZadigDebugContainerName, + Image: imageName, + Command: []string{"tail", "-f", "/dev/null"}, + ImagePullPolicy: corev1.PullAlways, + TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError, + }, + } +} + +func debugByEphemeralContainerLegacy(ctx context.Context, podClient corev1client.CoreV1Interface, pod *corev1.Pod, + debugContainer *corev1.EphemeralContainer) (*corev1.Pod, string, error) { + + patch, err := json.Marshal([]map[string]interface{}{{ + "op": "add", + "path": "/ephemeralContainers/-", + "value": debugContainer, + }}) + if err != nil { + return nil, "", fmt.Errorf("error creating JSON 6902 patch for old /ephemeralcontainers API: %s", err) + } + + result := podClient.RESTClient().Patch(types.JSONPatchType). + Namespace(pod.Namespace). + Resource("pods"). + Name(pod.Name). + SubResource("ephemeralcontainers"). + Body(patch). + Do(ctx) + if err := result.Error(); err != nil { + return nil, "", err + } + + newPod, err := podClient.Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{}) + if err != nil { + return nil, "", err + } + + return newPod, debugContainer.Name, nil +} + +func debugByEphemeralContainer(ctx context.Context, podClient corev1client.CoreV1Interface, pod *corev1.Pod, + debugContainer *corev1.EphemeralContainer) (*corev1.Pod, string, error) { + podJS, err := json.Marshal(pod) + if err != nil { + return nil, "", fmt.Errorf("error creating JSON for pod: %v", err) + } + + debugPod := pod.DeepCopy() + debugPod.Spec.EphemeralContainers = append(debugPod.Spec.EphemeralContainers, *debugContainer) + debugJS, err := json.Marshal(debugPod) + if err != nil { + return nil, "", fmt.Errorf("error creating JSON for debug container: %v", err) + } + + patch, err := strategicpatch.CreateTwoWayMergePatch(podJS, debugJS, pod) + if err != nil { + return nil, "", fmt.Errorf("error creating patch to add debug container: %v", err) + } + + pods := podClient.Pods(pod.Namespace) + result, err := pods.Patch(ctx, pod.Name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}, "ephemeralcontainers") + if err != nil { + return nil, "", fmt.Errorf("failed to patch: %s", err) + } + + return result, debugContainer.Name, nil +} diff --git a/pkg/microservice/aslan/core/environment/service/diff.go b/pkg/microservice/aslan/core/environment/service/diff.go index 0c493023793a1c910f167adc41f66755c3338315..b1f87f6a2e70d59327cdaacadc41321491bc88ee 100644 --- a/pkg/microservice/aslan/core/environment/service/diff.go +++ b/pkg/microservice/aslan/core/environment/service/diff.go @@ -88,11 +88,11 @@ func GetServiceDiff(envName, productName, serviceName string, log *zap.SugaredLo oldRender := &commonmodels.RenderSet{} newRender := &commonmodels.RenderSet{} if productInfo.Render != nil { - oldRender, err = commonservice.GetRenderSet(productInfo.Render.Name, productInfo.Render.Revision, log) + oldRender, err = commonservice.GetRenderSet(productInfo.Render.Name, productInfo.Render.Revision, false, envName, log) if err != nil { return resp, err } - newRender, err = commonservice.GetRenderSet(productInfo.Render.Name, 0, log) + newRender, err = commonservice.GetRenderSet(productInfo.Render.Name, 0, false, envName, log) if err != nil { return resp, err } diff --git a/pkg/microservice/aslan/core/environment/service/env.go b/pkg/microservice/aslan/core/environment/service/env.go index 477694aa946586f0d29b01e994103453367d256b..f56c2a541de280d88d67bab3fb695a4af8f7a917 100644 --- a/pkg/microservice/aslan/core/environment/service/env.go +++ b/pkg/microservice/aslan/core/environment/service/env.go @@ -18,6 +18,7 @@ package service import ( "go.uber.org/zap" + "k8s.io/client-go/informers" "sigs.k8s.io/controller-runtime/pkg/client" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" @@ -26,10 +27,10 @@ import ( ) type envHandle interface { - createGroup(envName, productName, username string, group []*commonmodels.ProductService, renderSet *commonmodels.RenderSet, kubeClient client.Client) error - listGroupServices(allServices []*commonmodels.ProductService, envName, productName string, kubeClient client.Client, productInfo *commonmodels.Product) []*commonservice.ServiceResp + createGroup(envName, productName, username string, group []*commonmodels.ProductService, renderSet *commonmodels.RenderSet, inf informers.SharedInformerFactory, kubeClient client.Client) error + listGroupServices(allServices []*commonmodels.ProductService, envName, productName string, informer informers.SharedInformerFactory, productInfo *commonmodels.Product) []*commonservice.ServiceResp updateService(args *SvcOptArgs) error - queryServiceStatus(namespace, envName, productName string, serviceTmpl *commonmodels.Service, kubeClient client.Client) (string, string, []string) + queryServiceStatus(namespace, envName, productName string, serviceTmpl *commonmodels.Service, kubeClient informers.SharedInformerFactory) (string, string, []string) } func envHandleFunc(projectType string, log *zap.SugaredLogger) envHandle { diff --git a/pkg/microservice/aslan/core/environment/service/environment.go b/pkg/microservice/aslan/core/environment/service/environment.go index 43e3675ba09ec37191173dcec18429f112e4e14a..3ecf901f83e65973f6e5ee2cff9ecf8ef98c26d1 100644 --- a/pkg/microservice/aslan/core/environment/service/environment.go +++ b/pkg/microservice/aslan/core/environment/service/environment.go @@ -22,54 +22,68 @@ import ( "fmt" "path/filepath" "regexp" - "sort" "strings" "sync" "time" + "github.com/cenkalti/backoff/v4" "github.com/hashicorp/go-multierror" helmclient "github.com/mittwald/go-helm-client" "github.com/pkg/errors" + "go.mongodb.org/mongo-driver/bson/primitive" "go.uber.org/zap" + "helm.sh/helm/v3/pkg/release" "helm.sh/helm/v3/pkg/releaseutil" "helm.sh/helm/v3/pkg/strvals" + versionedclient "istio.io/client-go/pkg/clientset/versioned" appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/informers" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/yaml" "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" + templatemodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + mongotemplate "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/collaboration" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" "github.com/koderover/zadig/pkg/shared/kube/wrapper" - "github.com/koderover/zadig/pkg/shared/poetry" e "github.com/koderover/zadig/pkg/tool/errors" helmtool "github.com/koderover/zadig/pkg/tool/helmclient" "github.com/koderover/zadig/pkg/tool/kube/getter" + "github.com/koderover/zadig/pkg/tool/kube/informer" "github.com/koderover/zadig/pkg/tool/kube/serializer" "github.com/koderover/zadig/pkg/tool/kube/updater" "github.com/koderover/zadig/pkg/tool/log" - "github.com/koderover/zadig/pkg/types/permission" + "github.com/koderover/zadig/pkg/types" "github.com/koderover/zadig/pkg/util" "github.com/koderover/zadig/pkg/util/converter" "github.com/koderover/zadig/pkg/util/fs" + yamlutil "github.com/koderover/zadig/pkg/util/yaml" ) const ( - Timeout = 60 - UpdateTypeSystem = "systemVar" - UpdateTypeEnv = "envVar" + Timeout = 60 +) + +const ( + usageScenarioCreateEnv = "createEnv" + usageScenarioUpdateEnv = "updateEnv" + usageScenarioUpdateRenderSet = "updateRenderSet" ) type EnvStatus struct { @@ -78,23 +92,55 @@ type EnvStatus struct { ErrMessage string `json:"err_message"` } +type EnvResp struct { + ProjectName string `json:"projectName"` + Status string `json:"status"` + Error string `json:"error"` + Name string `json:"name"` + UpdateBy string `json:"updateBy"` + UpdateTime int64 `json:"updateTime"` + IsPublic bool `json:"isPublic"` + ClusterName string `json:"clusterName"` + ClusterID string `json:"cluster_id"` + Production bool `json:"production"` + Source string `json:"source"` + RegistryID string `json:"registry_id"` + BaseRefs []string `json:"base_refs"` + BaseName string `json:"base_name"` + IsExisted bool `json:"is_existed"` + + // New Since v1.11.0 + ShareEnvEnable bool `json:"share_env_enable"` + ShareEnvIsBase bool `json:"share_env_is_base"` + ShareEnvBaseEnv string `json:"share_env_base_env"` +} + type ProductResp struct { - ID string `json:"id"` - ProductName string `json:"product_name"` - Namespace string `json:"namespace"` - Status string `json:"status"` - Error string `json:"error"` - EnvName string `json:"env_name"` - UpdateBy string `json:"update_by"` - UpdateTime int64 `json:"update_time"` - Services [][]string `json:"services"` - Render *commonmodels.RenderInfo `json:"render"` - Vars []*template.RenderKV `json:"vars"` - IsPublic bool `json:"isPublic"` - ClusterID string `json:"cluster_id,omitempty"` - RecycleDay int `json:"recycle_day"` - IsProd bool `json:"is_prod"` - Source string `json:"source"` + ID string `json:"id"` + ProductName string `json:"product_name"` + Namespace string `json:"namespace"` + Status string `json:"status"` + Error string `json:"error"` + EnvName string `json:"env_name"` + UpdateBy string `json:"update_by"` + UpdateTime int64 `json:"update_time"` + Services [][]string `json:"services"` + Render *commonmodels.RenderInfo `json:"render"` + Vars []*templatemodels.RenderKV `json:"vars"` + IsPublic bool `json:"isPublic"` + ClusterID string `json:"cluster_id,omitempty"` + ClusterName string `json:"cluster_name,omitempty"` + RecycleDay int `json:"recycle_day"` + IsProd bool `json:"is_prod"` + IsLocal bool `json:"is_local"` + IsExisted bool `json:"is_existed"` + Source string `json:"source"` + RegisterID string `json:"registry_id"` + + // New Since v1.11.0 + ShareEnvEnable bool `json:"share_env_enable"` + ShareEnvIsBase bool `json:"share_env_is_base"` + ShareEnvBaseEnv string `json:"share_env_base_env"` } type ProductParams struct { @@ -104,396 +150,134 @@ type ProductParams struct { PermissionUUIDs []string `json:"permissionUUIDs"` } +type EstimateValuesArg struct { + DefaultValues string `json:"defaultValues"` + OverrideYaml string `json:"overrideYaml"` + OverrideValues []*commonservice.KVPair `json:"overrideValues,omitempty"` +} + type EnvRenderChartArg struct { ChartValues []*commonservice.RenderChartArg `json:"chartValues"` } -type CreateHelmProductArg struct { - ProductName string `json:"productName"` - EnvName string `json:"envName"` - Namespace string `json:"namespace"` - ClusterID string `json:"clusterID"` - ChartValues []*commonservice.RenderChartArg `json:"chartValues"` +type EnvRendersetArg struct { + DefaultValues string `json:"defaultValues"` + ValuesData *commonservice.ValuesDataArgs `json:"valuesData"` + ChartValues []*commonservice.RenderChartArg `json:"chartValues"` + UpdateServiceTmpl bool `json:"updateServiceTmpl"` } -type UpdateMultiHelmProductArg struct { +type CreateHelmProductArg struct { ProductName string `json:"productName"` - EnvNames []string `json:"envNames"` + EnvName string `json:"envName"` + Namespace string `json:"namespace"` + ClusterID string `json:"clusterID"` + DefaultValues string `json:"defaultValues"` + ValuesData *commonservice.ValuesDataArgs `json:"valuesData"` + RegistryID string `json:"registry_id"` ChartValues []*commonservice.RenderChartArg `json:"chartValues"` - ReplacePolicy string `json:"replacePolicy"` // TODO logic not implemented + BaseEnvName string `json:"baseEnvName"` + BaseName string `json:"base_name,omitempty"` + IsExisted bool `json:"is_existed"` + // New Since v1.12.0 + ShareEnv commonmodels.ProductShareEnv `json:"share_env"` + // New Since v1.13.0 + EnvConfigs []*commonmodels.CreateUpdateCommonEnvCfgArgs `json:"env_configs"` } -func UpdateProductPublic(productName string, args *ProductParams, log *zap.SugaredLogger) error { - err := commonrepo.NewProductColl().UpdateIsPublic(args.EnvName, productName, args.IsPublic) - if err != nil { - log.Errorf("UpdateProductPublic error: %v", err) - return fmt.Errorf("UpdateProductPublic error: %v", err) - } - - poetryCtl := poetry.New(config.PoetryAPIServer(), config.PoetryAPIRootKey()) - if !args.IsPublic { //把公开设置成不公开 - _, err := poetryCtl.AddEnvRolePermission(productName, args.EnvName, args.PermissionUUIDs, args.RoleID, log) - if err != nil { - log.Errorf("UpdateProductPublic AddEnvRole error: %v", err) - return fmt.Errorf("UpdateProductPublic AddEnvRole error: %v", err) - } - return nil - } - //把不公开设成公开 删除原来环境绑定的角色 - _, err = poetryCtl.DeleteEnvRolePermission(productName, args.EnvName, log) - if err != nil { - log.Errorf("UpdateProductPublic DeleteEnvRole error: %v", err) - return fmt.Errorf("UpdateProductPublic DeleteEnvRole error: %v", err) - } - - return nil +type UpdateMultiHelmProductArg struct { + ProductName string `json:"productName"` + EnvNames []string `json:"envNames"` + ChartValues []*commonservice.RenderChartArg `json:"chartValues"` + DeletedServices []string `json:"deletedServices"` + ReplacePolicy string `json:"replacePolicy"` // TODO logic not implemented } -func ListProducts(productNameParam, envType string, userName string, userID int, superUser bool, log *zap.SugaredLogger) ([]*ProductResp, error) { - var ( - err error - testResp []*ProductResp - prodResp []*ProductResp - products = make([]*commonmodels.Product, 0) - productNameMap map[string][]int64 - productNamespaces = sets.NewString() - ) - resp := make([]*ProductResp, 0) +type RawYamlResp struct { + YamlContent string `json:"yamlContent"` +} - poetryCtl := poetry.New(config.PoetryAPIServer(), config.PoetryAPIRootKey()) +type ReleaseInstallParam struct { + ProductName string + Namespace string + ReleaseName string + MergedValues string + RenderChart *templatemodels.RenderChart + serviceObj *commonmodels.Service + DryRun bool +} - // 获取所有产品 - if superUser { - products, err = commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{Name: productNameParam}) - if err != nil { - log.Errorf("[%s] Collections.Product.List error: %v", userName, err) - return resp, e.ErrListEnvs.AddDesc(err.Error()) - } - } else { - //项目下所有公开环境 - publicProducts, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{IsPublic: true}) - if err != nil { - log.Errorf("Collection.Product.List List product error: %v", err) - return resp, e.ErrListProducts.AddDesc(err.Error()) - } - for _, publicProduct := range publicProducts { - if productNameParam == "" { - products = append(products, publicProduct) - productNamespaces.Insert(publicProduct.Namespace) - } else if publicProduct.ProductName == productNameParam { - products = append(products, publicProduct) - productNamespaces.Insert(publicProduct.Namespace) - } - } +type intervalExecutorHandler func(data *commonmodels.Service, isRetry bool, log *zap.SugaredLogger) error +type svcUpgradeFilter func(svc *commonmodels.ProductService) bool - productNameMap, err = poetryCtl.GetUserProject(userID, log) - if err != nil { - log.Errorf("Collection.Product.List GetUserProject error: %v", err) - return resp, e.ErrListProducts.AddDesc(err.Error()) - } - for productName, roleIDs := range productNameMap { - //用户关联角色所关联的环境 - for _, roleID := range roleIDs { - if roleID == setting.RoleOwnerID { - tmpProducts, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{Name: productName}) - if err != nil { - log.Errorf("Collection.Product.List Find product error: %v", err) - return resp, e.ErrListProducts.AddDesc(err.Error()) - } - for _, product := range tmpProducts { - if productNameParam == "" { - if !productNamespaces.Has(product.Namespace) { - products = append(products, product) - } - } else if product.ProductName == productNameParam { - if !productNamespaces.Has(product.Namespace) { - products = append(products, product) - } - } - } - } else { - productMap := make(map[string]int) - // 先列出环境-用户授权 - userEnvPermissionList, err := poetryCtl.GetUserEnvPermission(userID, log) - if err != nil { - log.Errorf("failed to get user env permission, err: %v", err) - return resp, err - } - for _, userEnvPermission := range userEnvPermissionList { - product, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{Name: productName, EnvName: userEnvPermission.EnvName}) - if err != nil { - log.Errorf("Collection.Product.List Find product error: %v", err) - continue - } - if productNameParam == "" { - if !productNamespaces.Has(product.Namespace) && userEnvPermission.PermissionUUID == permission.TestEnvListUUID { - products = append(products, product) - productNamespaces = productNamespaces.Insert(product.Namespace) - productMap[product.EnvName] = 1 - } - } else if product.ProductName == productNameParam { - if !productNamespaces.Has(product.Namespace) && userEnvPermission.PermissionUUID == permission.TestEnvManageUUID { - products = append(products, product) - productNamespaces = productNamespaces.Insert(product.Namespace) - productMap[product.EnvName] = 1 - } - } - } - // 再获取环境-角色授权 - roleEnvPermissions, err := poetryCtl.ListEnvRolePermission(productName, "", roleID, log) - if err != nil { - log.Errorf("Collection.Product.List ListRoleEnvs error: %v", err) - return resp, e.ErrListProducts.AddDesc(err.Error()) - } - for _, roleEnvPermission := range roleEnvPermissions { - product, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{Name: productName, EnvName: roleEnvPermission.EnvName}) - if err != nil { - log.Errorf("Collection.Product.List Find product error: %v", err) - continue - } - if productNameParam == "" { - if !productNamespaces.Has(product.Namespace) && roleEnvPermission.PermissionUUID == permission.TestEnvListUUID { - products = append(products, product) - productNamespaces.Insert(product.Namespace) - } - } else if product.ProductName == productNameParam { - if !productNamespaces.Has(product.Namespace) && roleEnvPermission.PermissionUUID == permission.TestEnvManageUUID { - products = append(products, product) - productNamespaces.Insert(product.Namespace) - } - } - } - } - } - } +func ListProducts(projectName string, envNames []string, log *zap.SugaredLogger) ([]*EnvResp, error) { + envs, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{Name: projectName, InEnvs: envNames, IsSortByProductName: true}) + if err != nil { + log.Errorf("Failed to list envs, err: %s", err) + return nil, e.ErrListEnvs.AddDesc(err.Error()) } - err = FillProductVars(products, log) + clusterMap := make(map[string]*commonmodels.K8SCluster) + clusters, err := commonrepo.NewK8SClusterColl().List(nil) if err != nil { - return resp, err + log.Errorf("Failed to list clusters in db, err: %s", err) + return nil, err } - for _, prod := range products { - product := &ProductResp{ - ID: prod.ID.Hex(), - ProductName: prod.ProductName, - EnvName: prod.EnvName, - Namespace: prod.Namespace, - Vars: prod.Vars[:], - IsPublic: prod.IsPublic, - ClusterID: prod.ClusterID, - UpdateTime: prod.UpdateTime, - UpdateBy: prod.UpdateBy, - RecycleDay: prod.RecycleDay, - Render: prod.Render, - Source: prod.Source, - } - - if prod.ClusterID != "" { - cluster, _ := commonrepo.NewK8SClusterColl().Get(prod.ClusterID) - if cluster != nil && cluster.Production { - product.IsProd = true - operatorPerm := poetryCtl.HasOperatePermission(prod.ProductName, permission.ProdEnvManageUUID, userID, superUser, log) - viewPerm := poetryCtl.HasOperatePermission(prod.ProductName, permission.ProdEnvListUUID, userID, superUser, log) - if envType == "" && (operatorPerm || viewPerm) { - prodResp = append(prodResp, product) - } else if envType == setting.ProdENV { - prodResp = append(prodResp, product) - } - } else if cluster != nil && !cluster.Production { - product.IsProd = false - testResp = append(testResp, product) - } - } else { - product.IsProd = false - testResp = append(testResp, product) - } - } - switch envType { - case setting.ProdENV: - resp = append(resp, prodResp...) - case setting.TestENV: - resp = append(resp, testResp...) - default: - resp = append(resp, prodResp...) - resp = append(resp, testResp...) + for _, cls := range clusters { + clusterMap[cls.ID.Hex()] = cls } - sort.SliceStable(resp, func(i, j int) bool { return resp[i].ProductName < resp[j].ProductName }) - - return resp, nil -} - -type ListProductsRespV2 struct { - ClusterName string `json:"clusterName"` - Production bool `json:"production"` - Name string `json:"name"` - ProjectName string `json:"projectName"` - Source string `json:"source"` -} - -// Args: projectName, which is formerly known as productName, is the primary key of the project in our system -func ListProductsV2(projectName, envFilter string, userName string, userID int, superUser bool, log *zap.SugaredLogger) ([]*ListProductsRespV2, error) { - var ( - err error - testResp []*ListProductsRespV2 - prodResp []*ListProductsRespV2 - products = make([]*commonmodels.Product, 0) - productNameMap map[string][]int64 - productNamespaces = sets.NewString() - ) - ret := make([]*ListProductsRespV2, 0) - - poetryCtl := poetry.New(config.PoetryAPIServer(), config.PoetryAPIRootKey()) - - // 获取所有产品 - if superUser { - products, err = commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{Name: projectName}) - if err != nil { - log.Errorf("[%s] Collections.Product.List error: %v", userName, err) - return ret, e.ErrListEnvs.AddDesc(err.Error()) - } - } else { - //项目下所有公开环境 - publicProducts, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{IsPublic: true}) - if err != nil { - log.Errorf("Collection.Product.List List product error: %v", err) - return ret, e.ErrListProducts.AddDesc(err.Error()) - } - for _, publicProduct := range publicProducts { - if projectName == "" { - products = append(products, publicProduct) - productNamespaces.Insert(publicProduct.Namespace) - } else if publicProduct.ProductName == projectName { - products = append(products, publicProduct) - productNamespaces.Insert(publicProduct.Namespace) - } - } - - productNameMap, err = poetryCtl.GetUserProject(userID, log) - if err != nil { - log.Errorf("Collection.Product.List GetUserProject error: %v", err) - return ret, e.ErrListProducts.AddDesc(err.Error()) - } - for productName, roleIDs := range productNameMap { - //用户关联角色所关联的环境 - for _, roleID := range roleIDs { - if roleID == setting.RoleOwnerID { - tmpProducts, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{Name: productName}) - if err != nil { - log.Errorf("Collection.Product.List Find product error: %v", err) - return ret, e.ErrListProducts.AddDesc(err.Error()) - } - for _, product := range tmpProducts { - if projectName == "" { - if !productNamespaces.Has(product.Namespace) { - products = append(products, product) - } - } else if product.ProductName == projectName { - if !productNamespaces.Has(product.Namespace) { - products = append(products, product) - } - } - } - } else { - productMap := make(map[string]int) - // 先列出环境-用户授权 - userEnvPermissionList, err := poetryCtl.GetUserEnvPermission(userID, log) - if err != nil { - log.Errorf("failed to get user env permission, err: %v", err) - return ret, err - } - for _, userEnvPermission := range userEnvPermissionList { - product, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{Name: productName, EnvName: userEnvPermission.EnvName}) - if err != nil { - log.Errorf("Collection.Product.List Find product error: %v", err) - continue - } - if projectName == "" { - if !productNamespaces.Has(product.Namespace) && userEnvPermission.PermissionUUID == permission.TestEnvListUUID { - products = append(products, product) - productNamespaces = productNamespaces.Insert(product.Namespace) - productMap[product.EnvName] = 1 - } - } else if product.ProductName == projectName { - if !productNamespaces.Has(product.Namespace) && userEnvPermission.PermissionUUID == permission.TestEnvManageUUID { - products = append(products, product) - productNamespaces = productNamespaces.Insert(product.Namespace) - productMap[product.EnvName] = 1 - } - } - } - // 再获取环境-角色授权 - roleEnvPermissions, err := poetryCtl.ListEnvRolePermission(productName, "", roleID, log) - if err != nil { - log.Errorf("Collection.Product.List ListRoleEnvs error: %v", err) - return ret, e.ErrListProducts.AddDesc(err.Error()) - } - for _, roleEnvPermission := range roleEnvPermissions { - product, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{Name: productName, EnvName: roleEnvPermission.EnvName}) - if err != nil { - log.Errorf("Collection.Product.List Find product error: %v", err) - continue - } - if projectName == "" { - if !productNamespaces.Has(product.Namespace) && roleEnvPermission.PermissionUUID == permission.TestEnvListUUID { - products = append(products, product) - productNamespaces.Insert(product.Namespace) - } - } else if product.ProductName == projectName { - if !productNamespaces.Has(product.Namespace) && roleEnvPermission.PermissionUUID == permission.TestEnvManageUUID { - products = append(products, product) - productNamespaces.Insert(product.Namespace) - } - } - } - } - } - } + var res []*EnvResp + reg, _, err := commonservice.FindDefaultRegistry(false, log) + if err != nil { + log.Errorf("FindDefaultRegistry error: %v", err) + return nil, err } - for _, product := range products { - item := &ListProductsRespV2{ - Name: product.EnvName, - ProjectName: product.ProductName, - Source: product.Source, - } - if product.ClusterID != "" { - cluster, _ := commonrepo.NewK8SClusterColl().Get(product.ClusterID) - if cluster != nil && cluster.Production { - item.Production = true - item.ClusterName = cluster.Name - operatorPerm := poetryCtl.HasOperatePermission(product.ProductName, permission.ProdEnvManageUUID, userID, superUser, log) - viewPerm := poetryCtl.HasOperatePermission(product.ProductName, permission.ProdEnvListUUID, userID, superUser, log) - if envFilter == "" && (operatorPerm || viewPerm) { - prodResp = append(prodResp, item) - } else if envFilter == setting.ProdENV { - prodResp = append(prodResp, item) - } - } else if cluster != nil && !cluster.Production { - item.Production = false - item.ClusterName = cluster.Name - testResp = append(testResp, item) - } - } else { - item.Production = false - testResp = append(testResp, item) - } + envCMMap, err := collaboration.GetEnvCMMap([]string{projectName}, log) + if err != nil { + return nil, err } - - switch envFilter { - case setting.ProdENV: - ret = append(ret, prodResp...) - case setting.TestENV: - ret = append(ret, testResp...) - default: - ret = append(ret, prodResp...) - ret = append(ret, testResp...) + for _, env := range envs { + clusterID := env.ClusterID + production := false + clusterName := "" + cluster, ok := clusterMap[clusterID] + if len(env.RegistryID) == 0 { + env.RegistryID = reg.ID.Hex() + } + if ok { + production = cluster.Production + clusterName = cluster.Name + } + var baseRefs []string + if cmSet, ok := envCMMap[collaboration.BuildEnvCMMapKey(env.ProductName, env.EnvName)]; ok { + baseRefs = append(baseRefs, cmSet.List()...) + } + res = append(res, &EnvResp{ + ProjectName: projectName, + Name: env.EnvName, + IsPublic: env.IsPublic, + IsExisted: env.IsExisted, + ClusterName: clusterName, + Source: env.Source, + Production: production, + Status: env.Status, + Error: env.Error, + UpdateTime: env.UpdateTime, + UpdateBy: env.UpdateBy, + RegistryID: env.RegistryID, + ClusterID: env.ClusterID, + BaseRefs: baseRefs, + BaseName: env.BaseName, + ShareEnvEnable: env.ShareEnv.Enable, + ShareEnvIsBase: env.ShareEnv.IsBase, + ShareEnvBaseEnv: env.ShareEnv.BaseEnv, + }) } - sort.SliceStable(ret, func(i, j int) bool { return ret[i].ProjectName < ret[j].ProjectName }) - - return ret, nil + return res, nil } func FillProductVars(products []*commonmodels.Product, log *zap.SugaredLogger) error { @@ -508,7 +292,7 @@ func FillProductVars(products []*commonmodels.Product, log *zap.SugaredLogger) e renderName = product.Render.Name revision = product.Render.Revision } - renderSet, err := commonservice.GetRenderSet(renderName, revision, log) + renderSet, err := commonservice.GetRenderSet(renderName, revision, false, product.EnvName, log) if err != nil { log.Errorf("Failed to find render set, productName: %s, namespace: %s, err: %s", product.ProductName, product.Namespace, err) return e.ErrGetRenderSet.AddDesc(err.Error()) @@ -547,7 +331,14 @@ func AutoCreateProduct(productName, envType, requestID string, log *zap.SugaredL var mutexAutoUpdate sync.RWMutex -func AutoUpdateProduct(envNames []string, productName string, userID int, superUser bool, requestID string, force bool, log *zap.SugaredLogger) ([]*EnvStatus, error) { +type UpdateEnv struct { + EnvName string `json:"env_name"` + ServiceNames []string `json:"service_names"` + UpdateType string `json:"update_type,omitempty"` + Vars []*template.RenderKV `json:"vars,omitempty"` +} + +func AutoUpdateProduct(args []*UpdateEnv, envNames []string, productName, requestID string, force bool, log *zap.SugaredLogger) ([]*EnvStatus, error) { mutexAutoUpdate.Lock() defer func() { mutexAutoUpdate.Unlock() @@ -555,24 +346,35 @@ func AutoUpdateProduct(envNames []string, productName string, userID int, superU envStatuses := make([]*EnvStatus, 0) - if !force { + project, err := templaterepo.NewProductColl().Find(productName) + if err != nil { + return nil, err + } + + if !force && project.ProductFeature != nil && project.ProductFeature.BasicFacility != setting.BasicFacilityCVM { modifiedByENV := make(map[string][]*serviceInfo) - for _, env := range envNames { - p, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{Name: productName, EnvName: env}) + for _, arg := range args { + p, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{Name: productName, EnvName: arg.EnvName}) if err != nil { - log.Errorf("Failed to get product %s in %s, error: %v", productName, env, err) + log.Errorf("Failed to get product %s in %s, error: %v", productName, arg.EnvName, err) continue } - kubeClient, err := kube.GetKubeClient(p.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), p.ClusterID) if err != nil { log.Errorf("Failed to get kube client for %s, error: %v", productName, err) continue } modifiedServices := getModifiedServiceFromObjectMetaList(kube.GetDirtyResources(p.Namespace, kubeClient)) - if len(modifiedServices) > 0 { - modifiedByENV[env] = modifiedServices + var specifyModifiedServices []*serviceInfo + for _, modifiedService := range modifiedServices { + if util.InStringArray(modifiedService.Name, arg.ServiceNames) { + specifyModifiedServices = append(specifyModifiedServices, modifiedService) + } + } + if len(specifyModifiedServices) > 0 { + modifiedByENV[arg.EnvName] = specifyModifiedServices } } if len(modifiedByENV) > 0 { @@ -584,7 +386,7 @@ func AutoUpdateProduct(envNames []string, productName string, userID int, superU } } - productsRevison, err := ListProductsRevision(productName, "", userID, superUser, log) + productsRevison, err := ListProductsRevision(productName, "", log) if err != nil { log.Errorf("AutoUpdateProduct ListProductsRevision err:%v", err) return envStatuses, err @@ -599,13 +401,8 @@ func AutoUpdateProduct(envNames []string, productName string, userID int, superU } } - for envName := range productMap { - productInfo, err := GetProduct(setting.SystemUser, envName, productName, log) - if err != nil { - log.Errorf("AutoUpdateProduct GetProduct err:%v", err) - return envStatuses, err - } - err = UpdateProductV2(envName, productName, setting.SystemUser, requestID, true, productInfo.Vars, log) + for _, arg := range args { + err = UpdateProductV2(arg.EnvName, productName, setting.SystemUser, requestID, arg.ServiceNames, force, arg.Vars, log) if err != nil { log.Errorf("AutoUpdateProduct UpdateProductV2 err:%v", err) return envStatuses, err @@ -631,7 +428,50 @@ func AutoUpdateProduct(envNames []string, productName string, userID int, superU } -func UpdateProduct(existedProd, updateProd *commonmodels.Product, renderSet *commonmodels.RenderSet, log *zap.SugaredLogger) (err error) { +// getServicesWithMaxRevision get all services template with max revision including involved shared services +func getServicesWithMaxRevision(projectName string) ([]*commonmodels.Service, error) { + // list services with max revision defined in project + allServices, err := commonrepo.NewServiceColl().ListMaxRevisions(&commonrepo.ServiceListOption{ProductName: projectName}) + if err != nil { + return nil, errors.Wrapf(err, "failed to find services in project %s", projectName) + } + + prodTmpl, err := templaterepo.NewProductColl().Find(projectName) + if err != nil { + return nil, errors.Wrapf(err, "failed to find project: %s", projectName) + } + + // list services with max revision of shared services + if prodTmpl.SharedServices != nil { + servicesByProject := make(map[string][]string) + for _, serviceInfo := range prodTmpl.SharedServices { + servicesByProject[serviceInfo.Owner] = append(servicesByProject[serviceInfo.Owner], serviceInfo.Name) + } + + for sourceProject, services := range servicesByProject { + inService := make([]*templatemodels.ServiceInfo, 0) + for _, serviceName := range services { + inService = append(inService, &templatemodels.ServiceInfo{ + Name: serviceName, + Owner: sourceProject, + }) + } + + sharedServices, err := commonrepo.NewServiceColl().ListMaxRevisions(&commonrepo.ServiceListOption{ + ProductName: sourceProject, + InServices: inService, + }) + if err != nil { + return nil, errors.Wrapf(err, "failed to find shared service templates, projectName: %s", sourceProject) + } + allServices = append(allServices, sharedServices...) + } + } + + return allServices, nil +} + +func UpdateProduct(serviceNames []string, existedProd, updateProd *commonmodels.Product, renderSet *commonmodels.RenderSet, log *zap.SugaredLogger) (err error) { // 设置产品新的renderinfo updateProd.Render = &commonmodels.RenderInfo{ Name: renderSet.Name, @@ -646,25 +486,17 @@ func UpdateProduct(existedProd, updateProd *commonmodels.Product, renderSet *com updateProd.Namespace = existedProd.Namespace var allServices []*commonmodels.Service - var allRenders []*commonmodels.RenderSet var prodRevs *ProductRevision - allServices, err = commonrepo.NewServiceColl().ListAllRevisions() - if err != nil { - log.Errorf("ListAllRevisions error: %v", err) - err = e.ErrUpdateEnv.AddDesc(err.Error()) - return - } - - // 获取所有渲染配置最新模板信息 - allRenders, err = commonrepo.NewRenderSetColl().ListAllRenders() + // list services with max revision of project + allServices, err = getServicesWithMaxRevision(productName) if err != nil { - log.Errorf("ListAllRevisions error: %v", err) + log.Errorf("ListAllRevisions error: %s", err) err = e.ErrUpdateEnv.AddDesc(err.Error()) return } - prodRevs, err = GetProductRevision(existedProd, allServices, allRenders, renderSet, log) + prodRevs, err = GetProductRevision(existedProd, allServices, log) if err != nil { err = e.ErrUpdateEnv.AddDesc(e.GetEnvRevErrMsg) return @@ -676,28 +508,50 @@ func UpdateProduct(existedProd, updateProd *commonmodels.Product, renderSet *com return } - kubeClient, err := kube.GetKubeClient(existedProd.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), existedProd.ClusterID) + if err != nil { + return e.ErrUpdateEnv.AddErr(err) + } + + restConfig, err := kubeclient.GetRESTConfig(config.HubServerAddress(), existedProd.ClusterID) + if err != nil { + return e.ErrUpdateEnv.AddErr(err) + } + + istioClient, err := versionedclient.NewForConfig(restConfig) if err != nil { return e.ErrUpdateEnv.AddErr(err) } + cls, err := kubeclient.GetKubeClientSet(config.HubServerAddress(), existedProd.ClusterID) + if err != nil { + log.Errorf("[%s][%s] error: %v", envName, namespace, err) + return e.ErrUpdateEnv.AddDesc(err.Error()) + + } + inf, err := informer.NewInformer(existedProd.ClusterID, namespace, cls) + if err != nil { + log.Errorf("[%s][%s] error: %v", envName, namespace, err) + return e.ErrUpdateEnv.AddDesc(err.Error()) + } + // 遍历产品环境和产品模板交叉对比的结果 // 四个状态：待删除，待添加，待更新，无需更新 - + var deletedServices []string // 1. 如果服务待删除：将产品模板中已经不存在，产品环境中待删除的服务进行删除。 for _, serviceRev := range prodRevs.ServiceRevisions { - if serviceRev.Updatable && serviceRev.Deleted { + if serviceRev.Updatable && serviceRev.Deleted && util.InStringArray(serviceRev.ServiceName, serviceNames) { log.Infof("[%s][P:%s][S:%s] start to delete service", envName, productName, serviceRev.ServiceName) //根据namespace: EnvName, selector: productName + serviceName来删除属于该服务的所有资源 selector := labels.Set{setting.ProductLabel: productName, setting.ServiceLabel: serviceRev.ServiceName}.AsSelector() - err = commonservice.DeleteResourcesAsync(namespace, selector, kubeClient, log) + err = commonservice.DeleteNamespacedResource(namespace, selector, existedProd.ClusterID, log) if err != nil { //删除失败仅记录失败日志 log.Errorf("delete resource of service %s error:%v", serviceRev.ServiceName, err) } - + deletedServices = append(deletedServices, serviceRev.ServiceName) clusterSelector := labels.Set{setting.ProductLabel: productName, setting.ServiceLabel: serviceRev.ServiceName, setting.EnvNameLabel: envName}.AsSelector() - err = commonservice.DeleteClusterResourceAsync(clusterSelector, kubeClient, log) + err = commonservice.DeleteClusterResource(clusterSelector, existedProd.ClusterID, log) if err != nil { //删除失败仅记录失败日志 log.Errorf("delete cluster resource of service %s error:%v", serviceRev.ServiceName, err) @@ -715,12 +569,11 @@ func UpdateProduct(existedProd, updateProd *commonmodels.Product, renderSet *com updateProd.Status = setting.ProductStatusUpdating updateProd.Services = updatedServices + updateProd.ShareEnv = existedProd.ShareEnv - log.Infof("[Namespace:%s][Product:%s]: update service orchestration in product. Status: %s", envName, productName, updateProd.Status) - if err = commonrepo.NewProductColl().Update(updateProd); err != nil { - log.Errorf("[Namespace:%s][Product:%s] Product.Update error: %v", envName, productName, err) - err = e.ErrUpdateEnv.AddErr(err) - return + if err := commonrepo.NewProductColl().UpdateStatus(envName, productName, setting.ProductStatusUpdating); err != nil { + log.Errorf("[%s][P:%s] Product.UpdateStatus error: %v", envName, productName, err) + return e.ErrUpdateEnv.AddDesc(e.UpdateEnvStatusErrMsg) } existedServices := existedProd.GetServiceMap() @@ -750,8 +603,6 @@ func UpdateProduct(existedProd, updateProd *commonmodels.Product, renderSet *com // 服务需要更新，需要upsert // 所有服务全部upsert一遍，确保所有服务起来 if svcRev.Updatable { - log.Infof("[Namespace:%s][Product:%s][Service:%s][IsNew:%v] upsert service", - envName, productName, svcRev.ServiceName, svcRev.New) service := &commonmodels.ProductService{ ServiceName: svcRev.ServiceName, @@ -763,7 +614,9 @@ func UpdateProduct(existedProd, updateProd *commonmodels.Product, renderSet *com service.Containers = svcRev.Containers service.Render = updateProd.Render - if svcRev.Type == setting.K8SDeployType { + if svcRev.Type == setting.K8SDeployType && util.InStringArray(service.ServiceName, serviceNames) { + log.Infof("[Namespace:%s][Product:%s][Service:%s][IsNew:%v] upsert service", + envName, productName, svcRev.ServiceName, svcRev.New) wg.Add(1) go func() { defer wg.Done() @@ -773,7 +626,7 @@ func UpdateProduct(existedProd, updateProd *commonmodels.Product, renderSet *com updateProd, service, existedServices[service.ServiceName], - renderSet, kubeClient, log) + renderSet, inf, kubeClient, istioClient, log) if err != nil { lock.Lock() switch e := err.(type) { @@ -800,7 +653,35 @@ func UpdateProduct(existedProd, updateProd *commonmodels.Product, renderSet *com err = e.ErrUpdateEnv.AddDesc(err.Error()) return } - err = commonrepo.NewProductColl().UpdateGroup(envName, productName, groupIndex, groupServices) + //merge new and old services + var updateGroup []*commonmodels.ProductService + newServiceMap := make(map[string]*commonmodels.ProductService) + for _, service := range groupServices { + newServiceMap[service.ServiceName] = service + } + oldServiceMap := make(map[string]*commonmodels.ProductService) + for _, existedGroupServices := range existedProd.Services { + for _, service := range existedGroupServices { + if util.InStringArray(service.ServiceName, deletedServices) { + continue + } + oldServiceMap[service.ServiceName] = service + if newService, ok := newServiceMap[service.ServiceName]; ok { + if util.InStringArray(service.ServiceName, serviceNames) { + updateGroup = append(updateGroup, newService) + } else { + updateGroup = append(updateGroup, service) + } + } + } + } + for _, newService := range groupServices { + if _, ok := oldServiceMap[newService.ServiceName]; !ok && !util.InStringArray(newService.ServiceName, deletedServices) && util.InStringArray(newService.ServiceName, serviceNames) { + updateGroup = append(updateGroup, newService) + } + } + + err = commonrepo.NewProductColl().UpdateGroup(envName, productName, groupIndex, updateGroup) if err != nil { log.Errorf("Failed to update collection - service group %d. Error: %v", groupIndex, err) err = e.ErrUpdateEnv.AddDesc(err.Error()) @@ -811,7 +692,32 @@ func UpdateProduct(existedProd, updateProd *commonmodels.Product, renderSet *com return nil } -func UpdateProductV2(envName, productName, user, requestID string, force bool, kvs []*template.RenderKV, log *zap.SugaredLogger) (err error) { +func UpdateProductRegistry(envName, productName, registryID string, log *zap.SugaredLogger) (err error) { + opt := &commonrepo.ProductFindOptions{EnvName: envName, Name: productName} + exitedProd, err := commonrepo.NewProductColl().Find(opt) + if err != nil { + log.Errorf("UpdateProductRegistry find product by envName:%s,error: %v", envName, err) + return e.ErrUpdateEnv.AddDesc(e.EnvNotFoundErrMsg) + } + err = commonrepo.NewProductColl().UpdateRegistry(envName, productName, registryID) + if err != nil { + log.Errorf("UpdateProductRegistry UpdateRegistry by envName:%s registryID:%s error: %v", envName, registryID, err) + return e.ErrUpdateEnv.AddErr(err) + } + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), exitedProd.ClusterID) + if err != nil { + return e.ErrUpdateEnv.AddErr(err) + } + err = ensureKubeEnv(exitedProd.Namespace, registryID, false, kubeClient, log) + + if err != nil { + log.Errorf("UpdateProductRegistry ensureKubeEnv by envName:%s,error: %v", envName, err) + return err + } + return nil +} + +func UpdateProductV2(envName, productName, user, requestID string, serviceNames []string, force bool, kvs []*templatemodels.RenderKV, log *zap.SugaredLogger) (err error) { // 根据产品名称和产品创建者到数据库中查找已有产品记录 opt := &commonrepo.ProductFindOptions{Name: productName, EnvName: envName} exitedProd, err := commonrepo.NewProductColl().Find(opt) @@ -820,42 +726,69 @@ func UpdateProductV2(envName, productName, user, requestID string, force bool, k return e.ErrUpdateEnv.AddDesc(e.EnvNotFoundErrMsg) } - kubeClient, err := kube.GetKubeClient(exitedProd.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), exitedProd.ClusterID) if err != nil { return e.ErrUpdateEnv.AddErr(err) } - - if !force { + project, err := templaterepo.NewProductColl().Find(productName) + if err != nil { + return err + } + if !force && project.ProductFeature != nil && project.ProductFeature.BasicFacility != setting.BasicFacilityCVM { modifiedServices := getModifiedServiceFromObjectMetaList(kube.GetDirtyResources(exitedProd.Namespace, kubeClient)) - if len(modifiedServices) > 0 { - data, err := json.Marshal(modifiedServices) + var specifyModifiedServices []*serviceInfo + for _, modifiedService := range modifiedServices { + if util.InStringArray(modifiedService.Name, serviceNames) { + specifyModifiedServices = append(specifyModifiedServices, modifiedService) + } + } + if len(specifyModifiedServices) > 0 { + data, err := json.Marshal(specifyModifiedServices) if err != nil { log.Errorf("Marshal failure: %v", err) } + log.Errorf("the following services are modified since last update: %s", data) return fmt.Errorf("the following services are modified since last update: %s", data) } } - err = ensureKubeEnv(exitedProd.Namespace, kubeClient, log) - - if err != nil { - log.Errorf("[%s][P:%s] service.UpdateProductV2 create kubeEnv error: %v", envName, productName, err) - return err + //TODO:The host update environment cannot remove deleted services + if !force && project.ProductFeature != nil && project.ProductFeature.BasicFacility == setting.BasicFacilityCVM { + services, err := commonrepo.NewServiceColl().ListMaxRevisionsAllSvcByProduct(productName) + if err != nil && !commonrepo.IsErrNoDocuments(err) { + log.Errorf("ListMaxRevisionsAllSvcByProduct: %s", err) + return fmt.Errorf("ListMaxRevisionsAllSvcByProduct: %s", err) + } + if services != nil { + svcNames := make([]string, len(services)) + for _, svc := range services { + svcNames = append(svcNames, svc.ServiceName) + } + serviceNames = svcNames + } } - err = commonservice.CreateRenderSet( - &commonmodels.RenderSet{ - Name: exitedProd.Namespace, - EnvName: envName, - ProductTmpl: productName, - KVs: kvs, - }, - log, - ) + if project.ProductFeature != nil && project.ProductFeature.BasicFacility != setting.BasicFacilityCVM { + err = ensureKubeEnv(exitedProd.Namespace, exitedProd.RegistryID, exitedProd.ShareEnv.Enable, kubeClient, log) - if err != nil { - log.Errorf("[%s][P:%s] create renderset error: %v", envName, productName, err) - return e.ErrUpdateEnv.AddDesc(e.FindProductTmplErrMsg) + if err != nil { + log.Errorf("[%s][P:%s] service.UpdateProductV2 create kubeEnv error: %v", envName, productName, err) + return err + } + + err = commonservice.CreateRenderSetByMerge( + &commonmodels.RenderSet{ + Name: exitedProd.Namespace, + EnvName: envName, + ProductTmpl: productName, + KVs: kvs, + }, + log, + ) + if err != nil { + log.Errorf("[%s][P:%s] create renderset error: %v", envName, productName, err) + return e.ErrUpdateEnv.AddDesc(e.FindProductTmplErrMsg) + } } // 检查renderinfo是否为空(适配历史product) @@ -864,7 +797,7 @@ func UpdateProductV2(envName, productName, user, requestID string, force bool, k } // 检查renderset是否覆盖产品所有key - renderSet, err := commonservice.ValidateRenderSet(exitedProd.ProductName, exitedProd.Render.Name, nil, log) + renderSet, err := commonservice.ValidateRenderSet(exitedProd.ProductName, exitedProd.Render.Name, exitedProd.EnvName, nil, log) if err != nil { log.Errorf("[%s][P:%s] validate product renderset error: %v", envName, exitedProd.ProductName, err) return e.ErrUpdateEnv.AddDesc(err.Error()) @@ -873,7 +806,7 @@ func UpdateProductV2(envName, productName, user, requestID string, force bool, k log.Infof("[%s][P:%s] UpdateProduct", envName, productName) // 查找产品模板 - updateProd, err := GetInitProduct(productName, log) + updateProd, err := GetInitProduct(productName, types.GeneralEnv, false, "", log) if err != nil { log.Errorf("[%s][P:%s] GetProductTemplate error: %v", envName, productName, err) return e.ErrUpdateEnv.AddDesc(e.FindProductTmplErrMsg) @@ -894,7 +827,7 @@ func UpdateProductV2(envName, productName, user, requestID string, force bool, k } go func() { - err := UpdateProduct(exitedProd, updateProd, renderSet, log) + err := UpdateProduct(serviceNames, exitedProd, updateProd, renderSet, log) if err != nil { log.Errorf("[%s][P:%s] failed to update product %#v", envName, productName, err) // 发送更新产品失败消息给用户 @@ -930,51 +863,46 @@ func UpdateProductV2(envName, productName, user, requestID string, force bool, k return nil } -func CreateHelmProduct(userName, requestID string, args *CreateHelmProductArg, log *zap.SugaredLogger) error { - templateProduct, err := templaterepo.NewProductColl().Find(args.ProductName) - if err != nil || templateProduct == nil { - if err != nil { - log.Errorf("failed to query product %s, err %s ", args.ProductName, err.Error()) - } - return e.ErrCreateEnv.AddDesc(fmt.Sprintf("failed to query product %s ", args.ProductName)) - } - - err = commonservice.FillProductTemplateValuesYamls(templateProduct, log) - if err != nil { - return e.ErrCreateEnv.AddDesc(err.Error()) +// fill product services and chart infos and insert renderset data +func prepareHelmProductCreation(templateProduct *templatemodels.Product, productObj *commonmodels.Product, arg *CreateHelmProductArg, serviceTmplMap map[string]*commonmodels.Service, log *zap.SugaredLogger) error { + productObj.ChartInfos = make([]*templatemodels.RenderChart, 0) + // chart infos in template product + templateChartInfoMap := make(map[string]*templatemodels.RenderChart) + for _, tc := range templateProduct.ChartInfos { + templateChartInfoMap[tc.ServiceName] = tc } - productObj := &commonmodels.Product{ - ProductName: args.ProductName, - Revision: 1, - Enabled: false, - EnvName: args.EnvName, - UpdateBy: userName, - IsPublic: true, - ClusterID: args.ClusterID, - Namespace: commonservice.GetProductEnvNamespace(args.EnvName, args.ProductName, args.Namespace), - Source: setting.SourceFromHelm, - IsOpenSource: templateProduct.IsOpensource, - ChartInfos: templateProduct.ChartInfos, - IsForkedProduct: false, + // user custom chart values + cvMap := make(map[string]*templatemodels.RenderChart) + for _, singleCV := range arg.ChartValues { + tc, ok := templateChartInfoMap[singleCV.ServiceName] + if !ok { + return fmt.Errorf("failed to find chart info in product, serviceName: %s productName: %s", singleCV.ServiceName, templateProduct.ProjectName) + } + chartInfo := &templatemodels.RenderChart{} + singleCV.FillRenderChartModel(chartInfo, tc.ChartVersion) + chartInfo.ValuesYaml = tc.ValuesYaml + productObj.ChartInfos = append(productObj.ChartInfos, chartInfo) + cvMap[singleCV.ServiceName] = chartInfo } - allServiceInfoMap := templateProduct.AllServiceInfoMap() + // default values + defaultValuesYaml := arg.DefaultValues + // generate service group data + var serviceGroup [][]*commonmodels.ProductService for _, names := range templateProduct.Services { servicesResp := make([]*commonmodels.ProductService, 0) for _, serviceName := range names { - opt := &commonrepo.ServiceFindOption{ - ServiceName: serviceName, - ProductName: allServiceInfoMap[serviceName].Owner, - ExcludeStatus: setting.ProductStatusDeleting, + // only the services chosen by use can be applied into product + rc, ok := cvMap[serviceName] + if !ok { + continue } - serviceTmpl, err := commonrepo.NewServiceColl().Find(opt) - if err != nil { - errMsg := fmt.Sprintf("Can not find service with option %+v, error: %v", opt, err) - log.Error(errMsg) - return e.ErrCreateEnv.AddDesc(errMsg) + serviceTmpl, ok := serviceTmplMap[serviceName] + if !ok { + return e.ErrCreateEnv.AddDesc(fmt.Sprintf("failed to find service info in template_service, serviceName: %s", serviceName)) } serviceResp := &commonmodels.ProductService{ @@ -983,69 +911,231 @@ func CreateHelmProduct(userName, requestID string, args *CreateHelmProductArg, l Type: serviceTmpl.Type, Revision: serviceTmpl.Revision, } - if serviceTmpl.Type == setting.K8SDeployType || serviceTmpl.Type == setting.HelmDeployType { - serviceResp.Containers = make([]*commonmodels.Container, 0) - for _, c := range serviceTmpl.Containers { - container := &commonmodels.Container{ - Name: c.Name, - Image: c.Image, - ImagePath: c.ImagePath, - } - serviceResp.Containers = append(serviceResp.Containers, container) + serviceResp.Containers = make([]*commonmodels.Container, 0) + var err error + for _, c := range serviceTmpl.Containers { + image := c.Image + image, err = genImageFromYaml(c, rc.ValuesYaml, defaultValuesYaml, rc.GetOverrideYaml(), rc.OverrideValues) + if err != nil { + errMsg := fmt.Sprintf("genImageFromYaml product template %s,service name:%s,error:%s", productObj.ProductName, rc.ServiceName, err) + log.Error(errMsg) + return e.ErrCreateEnv.AddDesc(errMsg) + } + container := &commonmodels.Container{ + Name: c.Name, + ImageName: util.GetImageNameFromContainerInfo(c.ImageName, c.Name), + Image: image, + ImagePath: c.ImagePath, } + serviceResp.Containers = append(serviceResp.Containers, container) } servicesResp = append(servicesResp, serviceResp) } - productObj.Services = append(productObj.Services, servicesResp) + serviceGroup = append(serviceGroup, servicesResp) } + productObj.Services = serviceGroup - // extract values.yaml from request and save into db - serviceList, err := commonrepo.NewServiceColl().ListMaxRevisionsByProduct(args.ProductName) + // insert renderset info into db + err := commonservice.CreateHelmRenderSet(&commonmodels.RenderSet{ + Name: commonservice.GetProductEnvNamespace(arg.EnvName, arg.ProductName, arg.Namespace), + EnvName: arg.EnvName, + ProductTmpl: arg.ProductName, + UpdateBy: productObj.UpdateBy, + IsDefault: false, + DefaultValues: arg.DefaultValues, + ChartInfos: productObj.ChartInfos, + YamlData: geneYamlData(arg.ValuesData), + }, log) if err != nil { - log.Infof("query services from product: %s fail, error %s", args.ProductName, err.Error()) - return e.ErrCreateEnv.AddDesc("failed to query services") + log.Errorf("rennderset create fail when copy creating helm product, productName: %s,envname:%s,err:%s", arg.ProductName, arg.EnvName, err) + return e.ErrCreateEnv.AddDesc(fmt.Sprintf("failed to save chart values, productName: %s,envname:%s,err:%s", arg.ProductName, arg.EnvName, err)) } + return nil +} - serviceMap := make(map[string]*commonmodels.Service) - for _, singleService := range serviceList { - serviceMap[singleService.ServiceName] = singleService +func CreateHelmProduct(productName, userName, requestID string, args []*CreateHelmProductArg, log *zap.SugaredLogger) error { + templateProduct, err := templaterepo.NewProductColl().Find(productName) + if err != nil || templateProduct == nil { + if err != nil { + log.Errorf("failed to query product %s, err %s ", productName, err.Error()) + } + return e.ErrCreateEnv.AddDesc(fmt.Sprintf("failed to query product %s ", productName)) } - customChartValueMap := make(map[string]*commonservice.RenderChartArg) - for _, singleCV := range args.ChartValues { - customChartValueMap[singleCV.ServiceName] = singleCV + // fill all chart infos from product renderset + err = commonservice.FillProductTemplateValuesYamls(templateProduct, log) + if err != nil { + return e.ErrCreateEnv.AddDesc(err.Error()) } - for _, latestChart := range productObj.ChartInfos { - if singleCV, ok := customChartValueMap[latestChart.ServiceName]; ok { - yamlContent, err := generateValuesYaml(singleCV, log) - if err != nil { - return e.ErrCreateEnv.AddDesc(fmt.Sprintf("failed to get yaml content for service: %s, err %v", singleCV.ServiceName, err.Error())) - } - singleCV.ValuesYAML = yamlContent - singleCV.FillRenderChartModel(latestChart, latestChart.ChartVersion) - } + // prepare data + serviceTmpls, err := commonrepo.NewServiceColl().ListMaxRevisionsByProduct(productName) + if err != nil { + return e.ErrCreateEnv.AddErr(err) + } + templateServiceMap := make(map[string]*commonmodels.Service) + for _, svc := range serviceTmpls { + templateServiceMap[svc.ServiceName] = svc } - // insert renderset info into db - if len(productObj.ChartInfos) > 0 { - err = commonservice.CreateHelmRenderSet(&commonmodels.RenderSet{ - Name: commonservice.GetProductEnvNamespace(args.EnvName, args.ProductName, args.Namespace), - EnvName: args.EnvName, - ProductTmpl: args.ProductName, - UpdateBy: userName, - IsDefault: false, - ChartInfos: productObj.ChartInfos, - }, log) + errList := new(multierror.Error) + for _, arg := range args { + dataValid := true + for _, cv := range arg.ChartValues { + if _, ok := templateServiceMap[cv.ServiceName]; !ok { + dataValid = false + errList = multierror.Append(errList, fmt.Errorf("failed to find service tempalte, serviceName: %s", cv.ServiceName)) + break + } + } + if !dataValid { + continue + } + err = createSingleHelmProduct(templateProduct, requestID, userName, arg.RegistryID, arg, templateServiceMap, log) if err != nil { - log.Errorf("rennderset create fail when creating helm product, productName: %s", args.ProductName) - return e.ErrCreateEnv.AddDesc(fmt.Sprintf("failed to save chart values, productName: %s", args.ProductName)) + errList = multierror.Append(errList, err) } } + return errList.ErrorOrNil() +} + +func createSingleHelmProduct(templateProduct *templatemodels.Product, requestID, userName, registryID string, arg *CreateHelmProductArg, serviceTmplMap map[string]*commonmodels.Service, log *zap.SugaredLogger) error { + productObj := &commonmodels.Product{ + ProductName: templateProduct.ProductName, + Revision: 1, + Enabled: false, + EnvName: arg.EnvName, + UpdateBy: userName, + IsPublic: true, + ClusterID: arg.ClusterID, + Namespace: commonservice.GetProductEnvNamespace(arg.EnvName, arg.ProductName, arg.Namespace), + Source: setting.SourceFromHelm, + IsOpenSource: templateProduct.IsOpensource, + IsForkedProduct: false, + RegistryID: registryID, + IsExisted: arg.IsExisted, + EnvConfigs: arg.EnvConfigs, + ShareEnv: arg.ShareEnv, + } + // fill services and chart infos of product + err := prepareHelmProductCreation(templateProduct, productObj, arg, serviceTmplMap, log) + if err != nil { + return err + } return CreateProduct(userName, requestID, productObj, log) } +type YamlProductItem struct { + OldName string `json:"old_name"` + NewName string `json:"new_name"` + BaseName string `json:"base_name"` + Vars []*templatemodels.RenderKV `json:"vars"` +} +type CopyYamlProductArg struct { + Items []YamlProductItem `json:"items"` +} + +type HelmProductItem struct { + OldName string `json:"old_name"` + NewName string `json:"new_name"` + BaseName string `json:"base_name"` + DefaultValues string `json:"default_values"` + ChartValues []*commonservice.RenderChartArg `json:"chart_values"` +} + +type CopyHelmProductArg struct { + Items []HelmProductItem +} + +func BulkCopyHelmProduct(projectName, user, requestID string, arg CopyHelmProductArg, log *zap.SugaredLogger) error { + if len(arg.Items) == 0 { + return nil + } + var envs []string + for _, item := range arg.Items { + envs = append(envs, item.OldName) + } + products, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{ + Name: projectName, + InEnvs: envs, + }) + if err != nil { + return err + } + productMap := make(map[string]*commonmodels.Product) + for _, product := range products { + productMap[product.EnvName] = product + } + var args []*CreateHelmProductArg + for _, item := range arg.Items { + if item.OldName == item.NewName { + continue + } + if product, ok := productMap[item.OldName]; ok { + args = append(args, &CreateHelmProductArg{ + ProductName: projectName, + EnvName: item.NewName, + Namespace: projectName + "-" + "env" + "-" + item.NewName, + ClusterID: product.ClusterID, + DefaultValues: item.DefaultValues, + RegistryID: product.RegistryID, + BaseEnvName: product.BaseName, + BaseName: item.BaseName, + ChartValues: item.ChartValues, + }) + } else { + return fmt.Errorf("product:%s not exist", item.OldName) + } + } + return CopyHelmProduct(projectName, user, requestID, args, log) +} + +func BulkCopyYamlProduct(projectName, user, requestID string, arg CopyYamlProductArg, log *zap.SugaredLogger) error { + if len(arg.Items) == 0 { + return nil + } + + var envs []string + for _, item := range arg.Items { + envs = append(envs, item.OldName) + } + products, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{ + Name: projectName, + InEnvs: envs, + }) + if err != nil { + return err + } + productMap := make(map[string]*commonmodels.Product) + for _, product := range products { + productMap[product.EnvName] = product + } + + for _, item := range arg.Items { + if item.OldName == item.NewName { + continue + } + if product, ok := productMap[item.OldName]; ok { + newProduct := *product + newProduct.EnvName = item.NewName + newProduct.Vars = item.Vars + newProduct.Namespace = projectName + "-env-" + newProduct.EnvName + newProduct.Render.Name = newProduct.Namespace + util.Clear(&newProduct.ID) + newProduct.Render.Revision = 0 + newProduct.BaseName = item.BaseName + err = CreateProduct(user, requestID, &newProduct, log) + if err != nil { + return err + } + } else { + return fmt.Errorf("product:%s not exist", item.OldName) + } + } + return nil +} + // CreateProduct create a new product with its dependent stacks func CreateProduct(user, requestID string, args *commonmodels.Product, log *zap.SugaredLogger) (err error) { log.Infof("[%s][P:%s] CreateProduct", args.EnvName, args.ProductName) @@ -1053,120 +1143,668 @@ func CreateProduct(user, requestID string, args *commonmodels.Product, log *zap. return creator.Create(user, requestID, args, log) } +func CopyHelmProduct(productName, userName, requestID string, args []*CreateHelmProductArg, log *zap.SugaredLogger) error { + errList := new(multierror.Error) + templateProduct, err := templaterepo.NewProductColl().Find(productName) + if err != nil || templateProduct == nil { + if err != nil { + log.Errorf("failed to query product %s, err %s ", productName, err.Error()) + } + return e.ErrCreateEnv.AddDesc(fmt.Sprintf("failed to query product %s ", productName)) + } + serviceTmpls, err := commonrepo.NewServiceColl().ListMaxRevisionsByProduct(productName) + if err != nil { + return e.ErrCreateEnv.AddErr(err) + } + templateServiceMap := make(map[string]*commonmodels.Service) + for _, svc := range serviceTmpls { + templateServiceMap[svc.ServiceName] = svc + } + + // fill all chart infos from product renderset + err = commonservice.FillProductTemplateValuesYamls(templateProduct, log) + if err != nil { + return e.ErrCreateEnv.AddDesc(err.Error()) + } + + for _, arg := range args { + err := copySingleHelmProduct(templateProduct, productName, requestID, userName, arg, templateServiceMap, log) + if err != nil { + errList = multierror.Append(errList, err) + } + } + return errList.ErrorOrNil() +} + +func copySingleHelmProduct(templateProduct *templatemodels.Product, productName, requestID, userName string, arg *CreateHelmProductArg, serviceTmplMap map[string]*commonmodels.Service, log *zap.SugaredLogger) error { + productInfo, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: productName, + EnvName: arg.BaseEnvName, + }) + if err != nil { + return fmt.Errorf("failed to query base product info name :%s,envname:%s", productName, arg.BaseEnvName) + } + + sourceRendersetName := productInfo.Namespace + productInfo.ID = primitive.NilObjectID + productInfo.Revision = 1 + productInfo.EnvName = arg.EnvName + productInfo.UpdateBy = userName + productInfo.ClusterID = arg.ClusterID + productInfo.BaseName = arg.BaseName + productInfo.Namespace = commonservice.GetProductEnvNamespace(arg.EnvName, arg.ProductName, arg.Namespace) + productInfo.EnvConfigs = arg.EnvConfigs + + // merge chart infos, use chart info in product to override charts in template_project + sourceRenderSet, _, err := commonrepo.NewRenderSetColl().FindRenderSet(&commonrepo.RenderSetFindOption{ + Name: sourceRendersetName, + EnvName: arg.BaseName, + ProductTmpl: arg.ProductName, + }) + if err != nil { + return fmt.Errorf("failed to find source renderset: %s, err: %s", productInfo.Namespace, err) + } + sourceChartMap := make(map[string]*templatemodels.RenderChart) + for _, singleChart := range sourceRenderSet.ChartInfos { + sourceChartMap[singleChart.ServiceName] = singleChart + } + templateCharts := templateProduct.ChartInfos + templateProduct.ChartInfos = make([]*templatemodels.RenderChart, 0) + for _, chart := range templateCharts { + if chartFromSource, ok := sourceChartMap[chart.ServiceName]; ok { + templateProduct.ChartInfos = append(templateProduct.ChartInfos, chartFromSource) + } else { + templateProduct.ChartInfos = append(templateProduct.ChartInfos, chart) + } + } + + // fill services and chart infos of product + err = prepareHelmProductCreation(templateProduct, productInfo, arg, serviceTmplMap, log) + if err != nil { + return err + } + + // clear render info + productInfo.Render = nil + setServiceRender(productInfo) + + // insert renderset info into db + if len(productInfo.ChartInfos) > 0 { + err := commonservice.CreateHelmRenderSet(&commonmodels.RenderSet{ + Name: commonservice.GetProductEnvNamespace(arg.EnvName, arg.ProductName, arg.Namespace), + EnvName: arg.EnvName, + ProductTmpl: arg.ProductName, + UpdateBy: userName, + IsDefault: false, + DefaultValues: arg.DefaultValues, + YamlData: geneYamlData(arg.ValuesData), + ChartInfos: productInfo.ChartInfos, + }, log) + if err != nil { + log.Errorf("rennderset create fail when copy creating helm product, productName: %s,envname:%s,err:%s", arg.ProductName, arg.EnvName, err) + return e.ErrCreateEnv.AddDesc(fmt.Sprintf("failed to save chart values, productName: %s,envname:%s,err:%s", arg.ProductName, arg.EnvName, err)) + } + } + return CreateProduct(userName, requestID, productInfo, log) +} + func UpdateProductRecycleDay(envName, productName string, recycleDay int) error { return commonrepo.NewProductColl().UpdateProductRecycleDay(envName, productName, recycleDay) } -func UpdateHelmProduct(productName, envName, updateType, username, requestID string, overrideCharts []*commonservice.RenderChartArg, log *zap.SugaredLogger) error { +func buildContainerMap(cs []*models.Container) map[string]*models.Container { + containerMap := make(map[string]*models.Container) + for _, c := range cs { + containerMap[c.Name] = c + } + return containerMap +} + +func UpdateHelmProduct(productName, envName, username, requestID string, overrideCharts []*commonservice.RenderChartArg, deletedServices []string, log *zap.SugaredLogger) error { opt := &commonrepo.ProductFindOptions{Name: productName, EnvName: envName} productResp, err := commonrepo.NewProductColl().Find(opt) if err != nil { log.Errorf("GetProduct envName:%s, productName:%s, err:%+v", envName, productName, err) return e.ErrUpdateEnv.AddDesc(err.Error()) } - currentProductService := productResp.Services - // 查找产品模板 - updateProd, err := GetInitProduct(productName, log) + + // create product data from product template + templateProd, err := GetInitProduct(productName, types.GeneralEnv, false, "", log) if err != nil { log.Errorf("[%s][P:%s] GetProductTemplate error: %v", envName, productName, err) return e.ErrUpdateEnv.AddDesc(e.FindProductTmplErrMsg) } - productResp.Services = updateProd.Services - // 设置产品状态为更新中 + // set image and render to the value used on current environment + deletedSvcSet := sets.NewString(deletedServices...) + deletedSvcRevision := make(map[string]int64) + // services need to be created or updated + serviceNeedUpdateOrCreate := sets.NewString() + for _, chart := range overrideCharts { + serviceNeedUpdateOrCreate.Insert(chart.ServiceName) + } + + productServiceMap := productResp.GetServiceMap() + + // get deleted services map[serviceName]=>serviceRevision + for _, svc := range productServiceMap { + if deletedSvcSet.Has(svc.ServiceName) { + deletedSvcRevision[svc.ServiceName] = svc.Revision + } + } + + // use service definition from service template, but keep the image info + allServices := make([][]*commonmodels.ProductService, 0) + for _, svrs := range templateProd.Services { + svcGroup := make([]*commonmodels.ProductService, 0) + for _, svr := range svrs { + if deletedSvcSet.Has(svr.ServiceName) { + continue + } + ps, ok := productServiceMap[svr.ServiceName] + // only update or insert services + if !ok && !serviceNeedUpdateOrCreate.Has(svr.ServiceName) { + continue + } + + // existed service has nothing to update + if ok && !serviceNeedUpdateOrCreate.Has(svr.ServiceName) { + svcGroup = append(svcGroup, ps) + continue + } + + svcGroup = append(svcGroup, svr) + if ps == nil { + continue + } + + templateContainMap := buildContainerMap(svr.Containers) + prodContainMap := buildContainerMap(ps.Containers) + for name, container := range templateContainMap { + if pc, ok := prodContainMap[name]; ok { + container.Image = pc.Image + } + } + } + allServices = append(allServices, svcGroup) + } + productResp.Services = allServices + + // set status to updating if err := commonrepo.NewProductColl().UpdateStatus(envName, productName, setting.ProductStatusUpdating); err != nil { log.Errorf("[%s][P:%s] Product.UpdateStatus error: %v", envName, productName, err) return e.ErrUpdateEnv.AddDesc(e.UpdateEnvStatusErrMsg) } + //对比当前环境中的环境变量和默认的环境变量 go func() { - err := updateProductGroup(productName, envName, updateType, productResp, currentProductService, overrideCharts, log) + err := updateProductGroup(username, productName, envName, productResp, overrideCharts, deletedSvcRevision, log) if err != nil { log.Errorf("[%s][P:%s] failed to update product %#v", envName, productName, err) // 发送更新产品失败消息给用户 title := fmt.Sprintf("更新 [%s] 的 [%s] 环境失败", productName, envName) commonservice.SendErrorMessage(username, title, requestID, err, log) - // 设置产品状态 - log.Infof("[%s][P:%s] update status to => %s", envName, productName, setting.ProductStatusFailed) - if err2 := commonrepo.NewProductColl().UpdateStatus(envName, productName, setting.ProductStatusFailed); err2 != nil { - log.Errorf("[%s][P:%s] Product.UpdateStatus error: %v", envName, productName, err2) - return - } - log.Infof("[%s][P:%s] update error to => %s", envName, productName, err) - if err2 := commonrepo.NewProductColl().UpdateErrors(envName, productName, err.Error()); err2 != nil { - log.Errorf("[%s][P:%s] Product.UpdateErrors error: %v", envName, productName, err2) + productResp.Status = setting.ProductStatusFailed + if err = commonrepo.NewProductColl().UpdateStatusAndError(envName, productName, productResp.Status, err.Error()); err != nil { + log.Errorf("[%s][%s] Product.Update error: %v", envName, productName, err) return } } else { productResp.Status = setting.ProductStatusSuccess - - if err = commonrepo.NewProductColl().UpdateStatus(envName, productName, productResp.Status); err != nil { + if err = commonrepo.NewProductColl().UpdateStatusAndError(envName, productName, productResp.Status, ""); err != nil { log.Errorf("[%s][%s] Product.Update error: %v", envName, productName, err) return } - - if err = commonrepo.NewProductColl().UpdateErrors(envName, productName, ""); err != nil { - log.Errorf("[%s][P:%s] Product.UpdateErrors error: %v", envName, productName, err) - return - } } }() return nil } +func genImageFromYaml(c *commonmodels.Container, valuesYaml, defaultValues, overrideYaml, overrideValues string) (string, error) { + mergeYaml, err := helmtool.MergeOverrideValues(valuesYaml, defaultValues, overrideYaml, overrideValues) + if err != nil { + return "", err + } + mergedValuesYamlFlattenMap, err := converter.YamlToFlatMap([]byte(mergeYaml)) + if err != nil { + return "", err + } + imageRule := templatemodels.ImageSearchingRule{ + Repo: c.ImagePath.Repo, + Image: c.ImagePath.Image, + Tag: c.ImagePath.Tag, + } + image, err := commonservice.GeneImageURI(imageRule.GetSearchingPattern(), mergedValuesYamlFlattenMap) + if err != nil { + return "", err + } + return image, nil +} + +func prepareEstimatedData(productName, envName, serviceName, usageScenario, defaultValues string, log *zap.SugaredLogger) (string, string, error) { + var err error + templateService, err := commonrepo.NewServiceColl().Find(&commonrepo.ServiceFindOption{ + ServiceName: serviceName, + ProductName: productName, + Type: setting.HelmDeployType, + }) + if err != nil { + log.Errorf("failed to query service, name %s, err %s", serviceName, err) + return "", "", fmt.Errorf("failed to query service, name %s", serviceName) + } + + if usageScenario == usageScenarioCreateEnv { + return templateService.HelmChart.ValuesYaml, defaultValues, nil + } + + productInfo, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: productName, + EnvName: envName, + }) + if err != nil { + return "", "", fmt.Errorf("failed to query product info, name %s", envName) + } + + // find chart info from cur render set + opt := &commonrepo.RenderSetFindOption{ + Name: productInfo.Render.Name, + Revision: productInfo.Render.Revision, + EnvName: productInfo.EnvName, + ProductTmpl: productInfo.ProductName, + } + renderSet, err := commonrepo.NewRenderSetColl().Find(opt) + if err != nil { + log.Errorf("renderset Find error, productName:%s, envName:%s, err:%s", productInfo.ProductName, productInfo.EnvName, err) + return "", "", fmt.Errorf("failed to query renderset info, name %s", productInfo.Render.Name) + } + + // find target render chart from render set + var targetChart *templatemodels.RenderChart + for _, chart := range renderSet.ChartInfos { + if chart.ServiceName == serviceName { + targetChart = chart + break + } + } + + switch usageScenario { + case usageScenarioUpdateEnv: + imageRelatedKey := sets.NewString() + proSvcMap := productInfo.GetServiceMap() + proSvc := proSvcMap[serviceName] + if proSvc != nil { + curEnvService, err := commonrepo.NewServiceColl().Find(&commonrepo.ServiceFindOption{ + ServiceName: serviceName, + ProductName: productName, + Type: setting.HelmDeployType, + Revision: proSvc.Revision, + }) + if err != nil { + log.Errorf("failed to query service, name %s, Revision %d,err %s", serviceName, proSvc.Revision, err) + return "", "", fmt.Errorf("failed to query service, name %s,Revision %d,err %s", serviceName, proSvc.Revision, err) + } + L: + for _, curSvcContainer := range curEnvService.Containers { + if checkServiceImageUpdated(curSvcContainer, proSvc) { + for _, container := range templateService.Containers { + if curSvcContainer.Name == container.Name && container.ImagePath != nil { + imageRelatedKey.Insert(container.ImagePath.Image, container.ImagePath.Repo, container.ImagePath.Tag) + continue L + } + } + } + } + } + curValuesYaml := "" + if targetChart != nil { // service has been applied into environment, use current values.yaml + curValuesYaml = targetChart.ValuesYaml + } + // merge environment values + mergedBs, err := overrideValues([]byte(curValuesYaml), []byte(templateService.HelmChart.ValuesYaml), imageRelatedKey) + if err != nil { + return "", "", errors.Wrapf(err, "failed to override values") + } + return string(mergedBs), renderSet.DefaultValues, nil + case usageScenarioUpdateRenderSet: + if targetChart == nil { + return "", "", fmt.Errorf("failed to find chart info, name: %s", serviceName) + } + return targetChart.ValuesYaml, renderSet.DefaultValues, nil + default: + return "", "", fmt.Errorf("unrecognized usageScenario:%s", usageScenario) + } +} + +func GeneEstimatedValues(productName, envName, serviceName, scene, format string, arg *EstimateValuesArg, log *zap.SugaredLogger) (interface{}, error) { + chartValues, defaultValues, err := prepareEstimatedData(productName, envName, serviceName, scene, arg.DefaultValues, log) + if err != nil { + return nil, e.ErrUpdateRenderSet.AddDesc(fmt.Sprintf("failed to prepare data, err %s", err)) + } + + tempArg := &commonservice.RenderChartArg{OverrideValues: arg.OverrideValues} + mergeValues, err := helmtool.MergeOverrideValues(chartValues, defaultValues, arg.OverrideYaml, tempArg.ToOverrideValueString()) + if err != nil { + return nil, e.ErrUpdateRenderSet.AddDesc(fmt.Sprintf("failed to merge values, err %s", err)) + } + + switch format { + case "flatMap": + mapData, err := converter.YamlToFlatMap([]byte(mergeValues)) + if err != nil { + return nil, e.ErrUpdateRenderSet.AddDesc(fmt.Sprintf("failed to generate flat map , err %s", err)) + } + return mapData, nil + default: + return &RawYamlResp{YamlContent: mergeValues}, nil + } +} + // check if override values or yaml content changes -func checkOverrideValuesChange(source *template.RenderChart, args *commonservice.RenderChartArg) bool { - tmpRenderCharts := &template.RenderChart{} - args.FillRenderChartModel(tmpRenderCharts, "") - if source.OverrideValues != tmpRenderCharts.OverrideValues || source.DiffOverrideYaml(tmpRenderCharts) { - return true +// return [need-Redeploy] and [need-SaveToDB] +func checkOverrideValuesChange(source *templatemodels.RenderChart, args *commonservice.RenderChartArg) (bool, bool) { + sourceArg := &commonservice.RenderChartArg{} + sourceArg.LoadFromRenderChartModel(source) + + same := sourceArg.DiffValues(args) + switch same { + case commonservice.Different: + return true, true + case commonservice.LogicSame: + return false, true + case commonservice.Same: + return false, false + } + return false, false +} + +func UpdateHelmProductDefaultValues(productName, envName, userName, requestID string, args *EnvRendersetArg, log *zap.SugaredLogger) error { + // validate if yaml content is legal + err := yaml.Unmarshal([]byte(args.DefaultValues), map[string]interface{}{}) + if err != nil { + return err } - return false + + product, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: productName, + EnvName: envName, + }) + if err != nil { + log.Errorf("UpdateHelmProductRenderset GetProductEnv envName:%s productName: %s error, error msg:%s", envName, productName, err) + return err + } + opt := &commonrepo.RenderSetFindOption{ + Name: product.Namespace, + EnvName: envName, + ProductTmpl: productName, + } + productRenderset, _, err := commonrepo.NewRenderSetColl().FindRenderSet(opt) + if err != nil || productRenderset == nil { + if err != nil { + log.Errorf("query renderset fail when updating helm product:%s render charts, err %s", productName, err.Error()) + } + return e.ErrUpdateEnv.AddDesc(fmt.Sprintf("failed to query renderset for envirionment: %s", envName)) + } + + equal, err := yamlutil.Equal(productRenderset.DefaultValues, args.DefaultValues) + if err != nil { + return e.ErrUpdateEnv.AddErr(fmt.Errorf("failed to unmarshal default values in renderset, err: %s", err)) + } + + productRenderset.DefaultValues = args.DefaultValues + productRenderset.YamlData = geneYamlData(args.ValuesData) + + updatedRcList := make([]*templatemodels.RenderChart, 0) + if !equal { + for _, curRenderChart := range productRenderset.ChartInfos { + updatedRcList = append(updatedRcList, curRenderChart) + } + } + + err = UpdateHelmProductVariable(productName, envName, userName, requestID, updatedRcList, productRenderset, log) + if err != nil { + return err + } + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), product.ClusterID) + if err != nil { + log.Errorf("UpdateHelmProductRenderset GetKubeClient error, error msg:%s", err) + return err + } + return ensureKubeEnv(product.Namespace, product.RegistryID, false, kubeClient, log) +} + +func UpdateHelmProductCharts(productName, envName, userName, requestID string, args *EnvRendersetArg, log *zap.SugaredLogger) error { + if len(args.ChartValues) == 0 { + return nil + } + + product, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: productName, + EnvName: envName, + }) + if err != nil { + log.Errorf("UpdateHelmProductRenderset GetProductEnv envName:%s productName: %s error, error msg:%s", envName, productName, err) + return err + } + opt := &commonrepo.RenderSetFindOption{ + Name: product.Namespace, + EnvName: envName, + ProductTmpl: productName, + } + productRenderset, _, err := commonrepo.NewRenderSetColl().FindRenderSet(opt) + if err != nil || productRenderset == nil { + if err != nil { + log.Errorf("query renderset fail when updating helm product:%s render charts, err %s", productName, err) + } + return e.ErrUpdateEnv.AddDesc(fmt.Sprintf("failed to query renderset for envirionment: %s", envName)) + } + + requestValueMap := make(map[string]*commonservice.RenderChartArg) + for _, arg := range args.ChartValues { + requestValueMap[arg.ServiceName] = arg + } + + valuesInRenderset := make(map[string]*templatemodels.RenderChart) + for _, rc := range productRenderset.ChartInfos { + valuesInRenderset[rc.ServiceName] = rc + } + + updatedRcMap := make(map[string]*templatemodels.RenderChart) + changedCharts := make([]*commonservice.RenderChartArg, 0) + + // update override values + for serviceName, arg := range requestValueMap { + arg.EnvName = envName + rcValues, ok := valuesInRenderset[serviceName] + if !ok { + log.Errorf("failed to find current chart values for service: %s", serviceName) + return e.ErrUpdateEnv.AddDesc(fmt.Sprintf("failed to find current chart values for service: %s", serviceName)) + } + + arg.FillRenderChartModel(rcValues, rcValues.ChartVersion) + changedCharts = append(changedCharts, arg) + updatedRcMap[serviceName] = rcValues + } + + // update service to latest revision acts like update service templates + if args.UpdateServiceTmpl { + updateEnvArg := &UpdateMultiHelmProductArg{ + ProductName: productName, + EnvNames: []string{envName}, + ChartValues: changedCharts, + } + _, err = UpdateMultipleHelmEnv(requestID, userName, updateEnvArg, log) + return err + } + + rcList := make([]*templatemodels.RenderChart, 0) + for _, rc := range updatedRcMap { + rcList = append(rcList, rc) + } + + return UpdateHelmProductVariable(productName, envName, userName, requestID, rcList, productRenderset, log) +} + +func geneYamlData(args *commonservice.ValuesDataArgs) *templatemodels.CustomYaml { + if args == nil { + return nil + } + var repoData *models.CreateFromRepo = nil + if args.GitRepoConfig != nil { + repoData = &models.CreateFromRepo{ + GitRepoConfig: &templatemodels.GitRepoConfig{ + CodehostID: args.GitRepoConfig.CodehostID, + Owner: args.GitRepoConfig.Owner, + Namespace: args.GitRepoConfig.Namespace, + Repo: args.GitRepoConfig.Repo, + Branch: args.GitRepoConfig.Branch, + }, + } + if len(args.GitRepoConfig.ValuesPaths) > 0 { + repoData.LoadPath = args.GitRepoConfig.ValuesPaths[0] + } + args.YamlSource = setting.SourceFromGitRepo + } + ret := &templatemodels.CustomYaml{ + Source: args.YamlSource, + AutoSync: args.AutoSync, + SourceDetail: repoData, + } + return ret } -func UpdateHelmProductRenderCharts(productName, envName, userName, requestID string, renderCharts []*commonservice.RenderChartArg, log *zap.SugaredLogger) error { +func SyncHelmProductEnvironment(productName, envName, requestID string, log *zap.SugaredLogger) error { + product, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: productName, + EnvName: envName, + }) + if err != nil { + log.Errorf("UpdateHelmProductRenderset GetProductEnv envName:%s productName: %s error, error msg:%s", envName, productName, err) + return err + } + opt := &commonrepo.RenderSetFindOption{ + Name: product.Namespace, + EnvName: envName, + ProductTmpl: productName, + } + productRenderset, _, err := commonrepo.NewRenderSetColl().FindRenderSet(opt) + if err != nil || productRenderset == nil { + if err != nil { + log.Errorf("query renderset fail when updating helm product:%s render charts, err %s", productName, err.Error()) + } + return e.ErrUpdateEnv.AddDesc(fmt.Sprintf("failed to query renderset for envirionment: %s", envName)) + } - renderSetName := commonservice.GetProductEnvNamespace(envName, productName, "") + updatedRCMap := make(map[string]*templatemodels.RenderChart) - opt := &commonrepo.RenderSetFindOption{Name: renderSetName} + changed, defaultValues, err := SyncYamlFromSource(productRenderset.YamlData, productRenderset.DefaultValues) + if err != nil { + return err + } + if changed { + productRenderset.DefaultValues = defaultValues + for _, curRenderChart := range productRenderset.ChartInfos { + updatedRCMap[curRenderChart.ServiceName] = curRenderChart + } + } + for _, chartInfo := range productRenderset.ChartInfos { + if chartInfo.OverrideYaml == nil { + continue + } + changed, values, err := SyncYamlFromSource(chartInfo.OverrideYaml, chartInfo.OverrideYaml.YamlContent) + if err != nil { + return err + } + if changed { + chartInfo.OverrideYaml.YamlContent = values + updatedRCMap[chartInfo.ServiceName] = chartInfo + } + } + if len(updatedRCMap) == 0 { + return nil + } + + // content of values.yaml changed, environment will be updated + updatedRcList := make([]*templatemodels.RenderChart, 0) + for _, updatedRc := range updatedRCMap { + updatedRcList = append(updatedRcList, updatedRc) + } + + err = UpdateHelmProductVariable(productName, envName, "cron", requestID, updatedRcList, productRenderset, log) + if err != nil { + return err + } + return err +} + +func UpdateHelmProductRenderset(productName, envName, userName, requestID string, args *EnvRendersetArg, log *zap.SugaredLogger) error { + product, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: productName, + EnvName: envName, + }) + if err != nil { + log.Errorf("UpdateHelmProductRenderset GetProductEnv envName:%s productName: %s error, error msg:%s", envName, productName, err) + return err + } + opt := &commonrepo.RenderSetFindOption{ + Name: product.Namespace, + EnvName: envName, + ProductTmpl: productName, + } productRenderset, _, err := commonrepo.NewRenderSetColl().FindRenderSet(opt) if err != nil || productRenderset == nil { if err != nil { - log.Infof("query renderset fail when updating helm product:%s render charts, err %s", productName, err.Error()) + log.Errorf("query renderset fail when updating helm product:%s render charts, err %s", productName, err.Error()) } return e.ErrUpdateEnv.AddDesc(fmt.Sprintf("failed to query renderset for envirionment: %s", envName)) } // render charts need to be updated - updatedRcs := make([]*template.RenderChart, 0) + updatedRcList := make([]*templatemodels.RenderChart, 0) + updatedRCMap := make(map[string]*templatemodels.RenderChart) - for _, requestRenderChart := range renderCharts { - yamlContent, err := generateValuesYaml(requestRenderChart, log) - if err != nil { - return e.ErrUpdateEnv.AddDesc(fmt.Sprintf("failed to get yaml content for service: %s, err %v", requestRenderChart.ServiceName, err.Error())) + // default values change + if args.DefaultValues != productRenderset.DefaultValues { + for _, curRenderChart := range productRenderset.ChartInfos { + updatedRCMap[curRenderChart.ServiceName] = curRenderChart } - requestRenderChart.ValuesYAML = yamlContent + productRenderset.DefaultValues = args.DefaultValues + } + productRenderset.YamlData = geneYamlData(args.ValuesData) + for _, requestRenderChart := range args.ChartValues { // update renderset info for _, curRenderChart := range productRenderset.ChartInfos { if curRenderChart.ServiceName != requestRenderChart.ServiceName { continue } - if !checkOverrideValuesChange(curRenderChart, requestRenderChart) { + if _, needSaveData := checkOverrideValuesChange(curRenderChart, requestRenderChart); !needSaveData { continue } requestRenderChart.FillRenderChartModel(curRenderChart, curRenderChart.ChartVersion) - updatedRcs = append(updatedRcs, curRenderChart) + updatedRCMap[curRenderChart.ServiceName] = curRenderChart break } } - return UpdateHelmProductVariable(productName, envName, userName, requestID, updatedRcs, productRenderset.ChartInfos, log) + for _, updatedRc := range updatedRCMap { + updatedRcList = append(updatedRcList, updatedRc) + } + + err = UpdateHelmProductVariable(productName, envName, userName, requestID, updatedRcList, productRenderset, log) + if err != nil { + return err + } + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), product.ClusterID) + if err != nil { + log.Errorf("UpdateHelmProductRenderset GetKubeClient error, error msg:%s", err) + return err + } + return ensureKubeEnv(product.Namespace, product.RegistryID, false, kubeClient, log) } -func UpdateHelmProductVariable(productName, envName, username, requestID string, updatedRcs, allRcs []*template.RenderChart, log *zap.SugaredLogger) error { +func UpdateHelmProductVariable(productName, envName, username, requestID string, updatedRcs []*templatemodels.RenderChart, renderset *commonmodels.RenderSet, log *zap.SugaredLogger) error { opt := &commonrepo.ProductFindOptions{Name: productName, EnvName: envName} productResp, err := commonrepo.NewProductColl().Find(opt) if err != nil { @@ -1181,11 +1819,13 @@ func UpdateHelmProductVariable(productName, envName, username, requestID string, if err = commonservice.CreateHelmRenderSet( &commonmodels.RenderSet{ - Name: productResp.Namespace, - EnvName: envName, - ProductTmpl: productName, - UpdateBy: username, - ChartInfos: allRcs, + Name: productResp.Namespace, + EnvName: envName, + ProductTmpl: productName, + UpdateBy: username, + DefaultValues: renderset.DefaultValues, + YamlData: renderset.YamlData, + ChartInfos: renderset.ChartInfos, }, log, ); err != nil { @@ -1193,79 +1833,71 @@ func UpdateHelmProductVariable(productName, envName, username, requestID string, return e.ErrUpdateEnv.AddDesc(e.FindProductTmplErrMsg) } + // update render info of product if productResp.Render == nil { productResp.Render = &commonmodels.RenderInfo{ProductTmpl: productResp.ProductName} } - - renderSet, err := FindHelmRenderSet(productResp.ProductName, productResp.Namespace, log) + renderSet, err := FindHelmRenderSet(productResp.ProductName, productResp.Namespace, envName, log) if err != nil { - log.Errorf("[%s][P:%s] find product renderset error: %v", productResp.EnvName, productResp.ProductName, err) - return e.ErrCreateEnv.AddDesc(err.Error()) + log.Errorf("[%s][P:%s] find product renderset error: %s", productResp.EnvName, productResp.ProductName, err) + return e.ErrUpdateEnv.AddDesc(err.Error()) } productResp.Render.Revision = renderSet.Revision + productResp.Render.Name = renderSet.Name - return updateHelmProductVariable(productResp, oldRenderVersion, username, requestID, log) -} + // update render used in product + err = commonrepo.NewProductColl().UpdateRender(envName, productResp.ProductName, productResp.Render) + if err != nil { + log.Errorf("[%s][P:%s] failed to update product renderset: %s", productResp.EnvName, productResp.ProductName, err) + return e.ErrUpdateEnv.AddErr(err) + } + + // only update renderset value to db, no need to upgrade chart release + if len(updatedRcs) == 0 { + return nil + } -func updateHelmProductVariable(productResp *commonmodels.Product, oldRenderVersion int64, userName, requestID string, log *zap.SugaredLogger) error { + return updateHelmProductVariable(productResp, renderSet, oldRenderVersion, username, requestID, log) +} +func updateHelmProductVariable(productResp *commonmodels.Product, renderset *commonmodels.RenderSet, oldRenderVersion int64, userName, requestID string, log *zap.SugaredLogger) error { envName, productName := productResp.EnvName, productResp.ProductName + restConfig, err := kube.GetRESTConfig(productResp.ClusterID) + if err != nil { + return e.ErrUpdateEnv.AddErr(err) + } - // 设置产品状态为更新中 + helmClient, err := helmtool.NewClientFromRestConf(restConfig, productResp.Namespace) + if err != nil { + return e.ErrUpdateEnv.AddErr(err) + } + + // set product status to updating if err := commonrepo.NewProductColl().UpdateStatus(envName, productName, setting.ProductStatusUpdating); err != nil { log.Errorf("[%s][P:%s] Product.UpdateStatus error: %v", envName, productName, err) return e.ErrUpdateEnv.AddDesc(e.UpdateEnvStatusErrMsg) } go func() { - err := updateProductVariable(productName, envName, productResp, log) + err := proceedHelmRelease(productName, envName, productResp, renderset, helmClient, nil, log) if err != nil { - log.Errorf("[%s][P:%s] failed to update product %#v", envName, productName, err) + log.Errorf("error occurred when upgrading services in env: %s/%s, err: %s ", productName, envName, err) // 发送更新产品失败消息给用户 title := fmt.Sprintf("更新 [%s] 的 [%s] 环境失败", productName, envName) commonservice.SendErrorMessage(userName, title, requestID, err, log) - - // 设置产品状态 - log.Infof("[%s][P:%s] update status to => %s", envName, productName, setting.ProductStatusFailed) - if err2 := commonrepo.NewProductColl().UpdateStatus(envName, productName, setting.ProductStatusFailed); err2 != nil { - log.Errorf("[%s][P:%s] Product.UpdateStatus error: %v", envName, productName, err2) - return - } - - log.Infof("[%s][P:%s] update error to => %s", envName, productName, err) - if err2 := commonrepo.NewProductColl().UpdateErrors(envName, productName, err.Error()); err2 != nil { - log.Errorf("[%s][P:%s] Product.UpdateErrors error: %v", envName, productName, err2) - return - } - - // If the update environment failed, Roll back to the previous render version and delete new revision - log.Infof("[%s][P:%s] roll back to previous render version: %d", envName, productName, oldRenderVersion) - if deleteRenderSetErr := commonrepo.NewRenderSetColl().DeleteRenderSet(productName, productResp.Render.Name, productResp.Render.Revision); deleteRenderSetErr != nil { - log.Errorf("[%s][P:%s] Product delete renderSet error: %v", envName, productName, deleteRenderSetErr) - } - productResp.Render.Revision = oldRenderVersion - if updateRendErr := commonrepo.NewProductColl().UpdateRender(envName, productName, productResp.Render); updateRendErr != nil { - log.Errorf("[%s][P:%s] Product update render error: %v", envName, productName, updateRendErr) - } - return } productResp.Status = setting.ProductStatusSuccess - if err = commonrepo.NewProductColl().UpdateStatus(envName, productName, productResp.Status); err != nil { + if err = commonrepo.NewProductColl().UpdateStatusAndError(envName, productName, productResp.Status, ""); err != nil { log.Errorf("[%s][%s] Product.Update error: %v", envName, productName, err) return } - - if err = commonrepo.NewProductColl().UpdateErrors(envName, productName, ""); err != nil { - log.Errorf("[%s][P:%s] Product.UpdateErrors error: %v", envName, productName, err) - return - } }() return nil } var mutexUpdateMultiHelm sync.RWMutex -func UpdateMultipleHelmEnv(userName, requestID string, userID int, superUser bool, args *UpdateMultiHelmProductArg, log *zap.SugaredLogger) ([]*EnvStatus, error) { +func UpdateMultipleHelmEnv(requestID, userName string, args *UpdateMultiHelmProductArg, log *zap.SugaredLogger) ([]*EnvStatus, error) { mutexUpdateMultiHelm.Lock() defer func() { mutexUpdateMultiHelm.Unlock() @@ -1274,7 +1906,7 @@ func UpdateMultipleHelmEnv(userName, requestID string, userID int, superUser boo envNames, productName := args.EnvNames, args.ProductName envStatuses := make([]*EnvStatus, 0) - productsRevision, err := ListProductsRevision(productName, "", userID, superUser, log) + productsRevision, err := ListProductsRevision(productName, "", log) if err != nil { log.Errorf("UpdateMultiHelmProduct ListProductsRevision err:%v", err) return envStatuses, err @@ -1286,38 +1918,35 @@ func UpdateMultipleHelmEnv(userName, requestID string, userID int, superUser boo if productRevision.ProductName != productName || !envNameSet.Has(productRevision.EnvName) { continue } - if !productRevision.Updatable { - continue - } + // NOTE. there is no need to check if product is updatable anymore productMap[productRevision.EnvName] = productRevision if len(productMap) == len(envNames) { break } } - serviceList, err := commonrepo.NewServiceColl().ListMaxRevisionsByProduct(args.ProductName) + // ensure related services exist in template services + templateProduct, err := templaterepo.NewProductColl().Find(productName) if err != nil { - log.Infof("query services from product: %s fail, error %s", args.ProductName, err.Error()) - return envStatuses, e.ErrUpdateEnv.AddDesc("failed to query services") + log.Errorf("failed to find template pruduct: %s, err: %s", productName, err) + return envStatuses, err } - - serviceMap := make(map[string]*commonmodels.Service) - for _, singleService := range serviceList { - serviceMap[singleService.ServiceName] = singleService + serviceNameSet := sets.NewString() + for _, svcGroup := range templateProduct.Services { + serviceNameSet.Insert(svcGroup...) } - - for _, requestRenderChart := range args.ChartValues { - yamlContent, err := generateValuesYaml(requestRenderChart, log) - if err != nil { - return envStatuses, e.ErrUpdateEnv.AddDesc(fmt.Sprintf("failed to get yaml content for service: %s, err %v", requestRenderChart.ServiceName, err.Error())) + for _, chartValue := range args.ChartValues { + if !serviceNameSet.Has(chartValue.ServiceName) { + return envStatuses, fmt.Errorf("failed to find service: %s in product template", chartValue.ServiceName) } - requestRenderChart.ValuesYAML = yamlContent } // extract values.yaml and update renderset - for envName, _ := range productMap { + for envName := range productMap { renderSet, _, err := commonrepo.NewRenderSetColl().FindRenderSet(&commonrepo.RenderSetFindOption{ - Name: commonservice.GetProductEnvNamespace(envName, productName, ""), + Name: commonservice.GetProductEnvNamespace(envName, productName, ""), + EnvName: envName, + ProductTmpl: productName, }) if err != nil || renderSet == nil { if err != nil { @@ -1326,7 +1955,7 @@ func UpdateMultipleHelmEnv(userName, requestID string, userID int, superUser boo return envStatuses, e.ErrUpdateEnv.AddDesc(fmt.Sprintf("failed to query renderset for env: %s", envName)) } - err = UpdateHelmProduct(productName, envName, UpdateTypeEnv, setting.SystemUser, requestID, args.ChartValues, log) + err = UpdateHelmProduct(productName, envName, userName, requestID, args.ChartValues, args.DeletedServices, log) if err != nil { log.Errorf("UpdateMultiHelmProduct UpdateProductV2 err:%v", err) return envStatuses, e.ErrUpdateEnv.AddDesc(err.Error()) @@ -1349,221 +1978,525 @@ func UpdateMultipleHelmEnv(userName, requestID string, userID int, superUser boo envStatuses = append(envStatuses, &EnvStatus{EnvName: productResp.EnvName, Status: productResp.Status}) } - return envStatuses, nil + return envStatuses, nil +} + +func GetProductInfo(username, envName, productName string, log *zap.SugaredLogger) (*commonmodels.Product, error) { + opt := &commonrepo.ProductFindOptions{Name: productName, EnvName: envName} + prod, err := commonrepo.NewProductColl().Find(opt) + if err != nil { + log.Errorf("[User:%s][EnvName:%s][Product:%s] Product.FindByOwner error: %v", username, envName, productName, err) + return nil, e.ErrGetEnv + } + + renderSetName := prod.Namespace + renderSetOpt := &commonrepo.RenderSetFindOption{Name: renderSetName, Revision: prod.Render.Revision, ProductTmpl: productName} + renderSet, err := commonrepo.NewRenderSetColl().Find(renderSetOpt) + if err != nil { + log.Errorf("find helm renderset[%s] error: %v", renderSetName, err) + return prod, nil + } + prod.ChartInfos = renderSet.ChartInfos + + return prod, nil +} + +func GetHelmChartVersions(productName, envName string, log *zap.SugaredLogger) ([]*commonmodels.HelmVersions, error) { + var ( + helmVersions = make([]*commonmodels.HelmVersions, 0) + chartInfoMap = make(map[string]*templatemodels.RenderChart) + ) + opt := &commonrepo.ProductFindOptions{Name: productName, EnvName: envName} + prod, err := commonrepo.NewProductColl().Find(opt) + if err != nil { + log.Errorf("[EnvName:%s][Product:%s] Product.FindByOwner error: %v", envName, productName, err) + return nil, e.ErrGetEnv + } + + prodTmpl, err := templaterepo.NewProductColl().Find(productName) + if err != nil { + log.Errorf("[EnvName:%s][Product:%s] get product template error: %v", envName, productName, err) + return nil, e.ErrGetEnv + } + + //当前环境的renderset + renderSetName := prod.Namespace + renderSetOpt := &commonrepo.RenderSetFindOption{Name: renderSetName, Revision: prod.Render.Revision, ProductTmpl: prod.ProductName} + renderSet, err := commonrepo.NewRenderSetColl().Find(renderSetOpt) + if err != nil { + log.Errorf("find helm renderset[%s] error: %v", renderSetName, err) + return helmVersions, err + } + for _, chartInfo := range renderSet.ChartInfos { + chartInfoMap[chartInfo.ServiceName] = chartInfo + } + + //当前环境内的服务信息 + prodServiceMap := prod.GetServiceMap() + + // all services + serviceListOpt := &commonrepo.ServiceListOption{ + ProductName: productName, + Type: setting.HelmDeployType, + } + for _, serviceGroup := range prodTmpl.Services { + for _, serviceName := range serviceGroup { + serviceListOpt.InServices = append(serviceListOpt.InServices, &templatemodels.ServiceInfo{ + Name: serviceName, + Owner: productName, + }) + } + } + //当前项目内最新的服务信息 + latestServices, err := commonrepo.NewServiceColl().ListMaxRevisions(serviceListOpt) + if err != nil { + log.Errorf("find service revision list error: %v", err) + return helmVersions, err + } + + for _, latestSvc := range latestServices { + if prodService, ok := prodServiceMap[latestSvc.ServiceName]; ok { + delete(prodServiceMap, latestSvc.ServiceName) + if latestSvc.Revision == prodService.Revision { + continue + } + helmVersion := &commonmodels.HelmVersions{ + ServiceName: latestSvc.ServiceName, + LatestVersion: latestSvc.HelmChart.Version, + LatestValuesYaml: latestSvc.HelmChart.ValuesYaml, + } + if chartInfo, ok := chartInfoMap[latestSvc.ServiceName]; ok { + helmVersion.CurrentVersion = chartInfo.ChartVersion + helmVersion.CurrentValuesYaml = chartInfo.ValuesYaml + } + helmVersions = append(helmVersions, helmVersion) + } else { // new service + helmVersion := &commonmodels.HelmVersions{ + ServiceName: latestSvc.ServiceName, + LatestVersion: latestSvc.HelmChart.Version, + LatestValuesYaml: latestSvc.HelmChart.ValuesYaml, + } + helmVersions = append(helmVersions, helmVersion) + } + } + + // deleted service + for _, prodService := range prodServiceMap { + helmVersion := &commonmodels.HelmVersions{ + ServiceName: prodService.ServiceName, + } + if chartInfo, ok := chartInfoMap[prodService.ServiceName]; ok { + helmVersion.CurrentVersion = chartInfo.ChartVersion + helmVersion.CurrentValuesYaml = chartInfo.ValuesYaml + } + helmVersions = append(helmVersions, helmVersion) + } + + return helmVersions, nil +} + +func DeleteProduct(username, envName, productName, requestID string, isDelete bool, log *zap.SugaredLogger) (err error) { + eventStart := time.Now().Unix() + productInfo, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{Name: productName, EnvName: envName}) + if err != nil { + log.Errorf("find product error: %v", err) + return err + } + + // delete informer's cache + informer.DeleteInformer(productInfo.ClusterID, productInfo.Namespace) + + envCMMap, err := collaboration.GetEnvCMMap([]string{productName}, log) + if err != nil { + return err + } + if cmSets, ok := envCMMap[collaboration.BuildEnvCMMapKey(productName, envName)]; ok { + return fmt.Errorf("this is a base environment, collaborations:%v is related", cmSets.List()) + } + + restConfig, err := kube.GetRESTConfig(productInfo.ClusterID) + if err != nil { + return e.ErrDeleteEnv.AddErr(err) + } + + istioClient, err := versionedclient.NewForConfig(restConfig) + if err != nil { + return e.ErrDeleteEnv.AddErr(err) + } + + err = commonrepo.NewProductColl().UpdateStatus(envName, productName, setting.ProductStatusDeleting) + if err != nil { + log.Errorf("[%s][%s] update product status error: %v", username, productInfo.Namespace, err) + return e.ErrDeleteEnv.AddDesc("更新环境状态失败: " + err.Error()) + } + + log.Infof("[%s] delete product %s", username, productInfo.Namespace) + commonservice.LogProductStats(username, setting.DeleteProductEvent, productName, requestID, eventStart, log) + + ctx := context.TODO() + switch productInfo.Source { + case setting.SourceFromHelm: + // Handles environment sharing related operations. + err = EnsureDeleteShareEnvConfig(ctx, productInfo, istioClient) + if err != nil { + log.Errorf("Failed to delete share env config for env %s of product %s: %s", productInfo.EnvName, productInfo.ProductName, err) + } + + err = commonrepo.NewProductColl().Delete(envName, productName) + if err != nil { + log.Errorf("Product.Delete error: %v", err) + } + + go func() { + errList := &multierror.Error{} + defer func() { + if errList.ErrorOrNil() != nil { + title := fmt.Sprintf("删除项目:[%s] 环境:[%s] 失败!", productName, envName) + commonservice.SendErrorMessage(username, title, requestID, errList.ErrorOrNil(), log) + _ = commonrepo.NewProductColl().UpdateStatus(envName, productName, setting.ProductStatusUnknown) + } else { + title := fmt.Sprintf("删除项目:[%s] 环境:[%s] 成功!", productName, envName) + content := fmt.Sprintf("namespace:%s", productInfo.Namespace) + commonservice.SendMessage(username, title, content, requestID, log) + } + }() + + if isDelete { + if hc, errHelmClient := helmtool.NewClientFromRestConf(restConfig, productInfo.Namespace); errHelmClient == nil { + for _, service := range productInfo.GetServiceMap() { + if err = UninstallServiceByName(hc, service.ServiceName, productInfo, service.Revision, true); err != nil { + log.Warnf("UninstallRelease for service %s err:%s", service.ServiceName, err) + errList = multierror.Append(errList, err) + } + } + } else { + log.Errorf("failed to get helmClient, err: %s", errHelmClient) + errList = multierror.Append(errList, e.ErrDeleteEnv.AddErr(errHelmClient)) + return + } + + s := labels.Set{setting.EnvCreatedBy: setting.EnvCreator}.AsSelector() + if err := commonservice.DeleteNamespaceIfMatch(productInfo.Namespace, s, productInfo.ClusterID, log); err != nil { + errList = multierror.Append(errList, e.ErrDeleteEnv.AddDesc(e.DeleteNamespaceErrMsg+": "+err.Error())) + return + } + } + }() + case setting.SourceFromExternal: + err = commonrepo.NewProductColl().Delete(envName, productName) + if err != nil { + log.Errorf("Product.Delete error: %v", err) + } + + tempProduct, err := mongotemplate.NewProductColl().Find(productName) + if err != nil { + log.Errorf("project not found error:%s", err) + } + if tempProduct.ProductFeature != nil && tempProduct.ProductFeature.CreateEnvType == setting.SourceFromExternal { + workloadStat, err := commonrepo.NewWorkLoadsStatColl().Find(productInfo.ClusterID, productInfo.Namespace) + if err != nil { + log.Errorf("workflowStat not found error:%s", err) + } + if workloadStat != nil { + workloadStat.Workloads = commonservice.FilterWorkloadsByEnv(workloadStat.Workloads, productInfo.EnvName) + if err := commonrepo.NewWorkLoadsStatColl().UpdateWorkloads(workloadStat); err != nil { + log.Errorf("update workloads fail error:%s", err) + } + } + + currentEnvServices, err := commonrepo.NewServiceColl().ListExternalWorkloadsBy(productName, envName) + if err != nil { + log.Errorf("failed to list external workload, error:%s", err) + } + + externalEnvServices, err := commonrepo.NewServicesInExternalEnvColl().List(&commonrepo.ServicesInExternalEnvArgs{ + ProductName: productName, + ExcludeEnvName: envName, + }) + if err != nil { + log.Errorf("failed to list external service, error:%s", err) + } + + externalEnvServiceM := make(map[string]bool) + for _, externalEnvService := range externalEnvServices { + externalEnvServiceM[externalEnvService.ServiceName] = true + } + + deleteServices := sets.NewString() + for _, currentEnvService := range currentEnvServices { + if _, isExist := externalEnvServiceM[currentEnvService.ServiceName]; !isExist { + deleteServices.Insert(currentEnvService.ServiceName) + } + } + err = commonrepo.NewServiceColl().BatchUpdateExternalServicesStatus(productName, "", setting.ProductStatusDeleting, deleteServices.List()) + if err != nil { + log.Errorf("UpdateStatus external services error:%s", err) + } + // delete services_in_external_env data + if err = commonrepo.NewServicesInExternalEnvColl().Delete(&commonrepo.ServicesInExternalEnvArgs{ + ProductName: productName, + EnvName: envName, + }); err != nil { + log.Errorf("remove services in external env error:%s", err) + } + } + case setting.SourceFromPM: + err = commonrepo.NewProductColl().Delete(envName, productName) + if err != nil { + log.Errorf("Product.Delete error: %v", err) + } + default: + go func() { + var err error + defer func() { + if err != nil { + title := fmt.Sprintf("删除项目:[%s] 环境:[%s] 失败!", productName, envName) + commonservice.SendErrorMessage(username, title, requestID, err, log) + _ = commonrepo.NewProductColl().UpdateStatus(envName, productName, setting.ProductStatusUnknown) + } else { + title := fmt.Sprintf("删除项目:[%s] 环境:[%s] 成功!", productName, envName) + content := fmt.Sprintf("namespace:%s", productInfo.Namespace) + commonservice.SendMessage(username, title, content, requestID, log) + } + }() + if isDelete { + // Delete Cluster level resources + err = commonservice.DeleteClusterResource(labels.Set{setting.ProductLabel: productName, setting.EnvNameLabel: envName}.AsSelector(), productInfo.ClusterID, log) + if err != nil { + err = e.ErrDeleteProduct.AddDesc(e.DeleteServiceContainerErrMsg + ": " + err.Error()) + return + } + + // Handles environment sharing related operations. + err = EnsureDeleteShareEnvConfig(ctx, productInfo, istioClient) + if err != nil { + log.Errorf("Failed to delete share env config: %s", err) + err = e.ErrDeleteProduct.AddDesc(e.DeleteVirtualServiceErrMsg + ": " + err.Error()) + return + } + + s := labels.Set{setting.EnvCreatedBy: setting.EnvCreator}.AsSelector() + if err1 := commonservice.DeleteNamespaceIfMatch(productInfo.Namespace, s, productInfo.ClusterID, log); err1 != nil { + err = e.ErrDeleteEnv.AddDesc(e.DeleteNamespaceErrMsg + ": " + err1.Error()) + return + } + } + err = commonrepo.NewProductColl().Delete(envName, productName) + if err != nil { + log.Errorf("Product.Delete error: %v", err) + } + }() + } + + return nil } -// UpdateMultiHelmProduct TODO need to be deprecated -func UpdateMultiHelmProduct(envNames []string, updateType, productName string, userID int, superUser bool, requestID string, log *zap.SugaredLogger) []*EnvStatus { - mutexUpdateMultiHelm.Lock() - defer func() { - mutexUpdateMultiHelm.Unlock() - }() - - envStatuses := make([]*EnvStatus, 0) - productsRevison, err := ListProductsRevision(productName, "", userID, superUser, log) +func DeleteProductServices(userName, requestID, envName, productName string, serviceNames []string, log *zap.SugaredLogger) (err error) { + productInfo, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{Name: productName, EnvName: envName}) if err != nil { - log.Errorf("UpdateMultiHelmProduct ListProductsRevision err:%v", err) - return envStatuses + log.Errorf("find product error: %v", err) + return err } - productMap := make(map[string]*ProductRevision) - for _, productRevison := range productsRevison { - if productRevison.ProductName == productName && sets.NewString(envNames...).Has(productRevison.EnvName) && productRevison.Updatable { - productMap[productRevison.EnvName] = productRevison - if len(productMap) == len(envNames) { - break - } - } + if getProjectType(productName) == setting.HelmDeployType { + return deleteHelmProductServices(userName, requestID, productInfo, serviceNames, log) } + return deleteK8sProductServices(productInfo, serviceNames, log) +} - for envName := range productMap { - err = UpdateHelmProduct(productName, envName, updateType, setting.SystemUser, requestID, nil, log) - if err != nil { - log.Errorf("UpdateMultiHelmProduct UpdateProductV2 err:%v", err) - return envStatuses - } +func deleteHelmProductServices(userName, requestID string, productInfo *commonmodels.Product, serviceNames []string, log *zap.SugaredLogger) error { + restConfig, err := kubeclient.GetRESTConfig(config.HubServerAddress(), productInfo.ClusterID) + if err != nil { + return e.ErrUpdateEnv.AddErr(err) } - - productResps := make([]*ProductResp, 0) - for _, envName := range envNames { - productResp, err := GetProduct(setting.SystemUser, envName, productName, log) - if err == nil && productResp != nil { - productResps = append(productResps, productResp) - } + helmClient, err := helmtool.NewClientFromRestConf(restConfig, productInfo.Namespace) + if err != nil { + return e.ErrUpdateEnv.AddErr(err) } - for _, productResp := range productResps { - if productResp.Error != "" { - envStatuses = append(envStatuses, &EnvStatus{EnvName: productResp.EnvName, Status: setting.ProductStatusFailed, ErrMessage: productResp.Error}) - continue - } - envStatuses = append(envStatuses, &EnvStatus{EnvName: productResp.EnvName, Status: productResp.Status}) + ctx := context.TODO() + kclient, err := kubeclient.GetKubeClient(config.HubServerAddress(), productInfo.ClusterID) + if err != nil { + return e.ErrUpdateEnv.AddErr(err) } - return envStatuses -} -func GetProductInfo(username, envName, productName string, log *zap.SugaredLogger) (*commonmodels.Product, error) { - opt := &commonrepo.ProductFindOptions{Name: productName, EnvName: envName} - prod, err := commonrepo.NewProductColl().Find(opt) + istioClient, err := versionedclient.NewForConfig(restConfig) if err != nil { - log.Errorf("[User:%s][EnvName:%s][Product:%s] Product.FindByOwner error: %v", username, envName, productName, err) - return nil, e.ErrGetEnv + return e.ErrUpdateEnv.AddErr(err) } - renderSetName := prod.Namespace - renderSetOpt := &commonrepo.RenderSetFindOption{Name: renderSetName, Revision: prod.Render.Revision} - renderSet, err := commonrepo.NewRenderSetColl().Find(renderSetOpt) + deleteServiceSet := sets.NewString(serviceNames...) + deletedSvcRevision := make(map[string]int64) + + for serviceGroupIndex, serviceGroup := range productInfo.Services { + var group []*commonmodels.ProductService + for _, service := range serviceGroup { + if !deleteServiceSet.Has(service.ServiceName) { + group = append(group, service) + } else { + deletedSvcRevision[service.ServiceName] = service.Revision + } + } + err := commonrepo.NewProductColl().UpdateGroup(productInfo.EnvName, productInfo.ProductName, serviceGroupIndex, group) + if err != nil { + log.Errorf("update product error: %v", err) + return err + } + } + renderset, err := commonrepo.NewRenderSetColl().Find(&commonrepo.RenderSetFindOption{ + Name: productInfo.Namespace, + EnvName: productInfo.EnvName, + ProductTmpl: productInfo.ProductName, + }) if err != nil { - log.Errorf("find helm renderset[%s] error: %v", renderSetName, err) - return prod, nil + log.Errorf("get renderSet error: %v", err) + return err } - prod.ChartInfos = renderSet.ChartInfos + rcs := make([]*template.RenderChart, 0) + for _, v := range renderset.ChartInfos { + if !deleteServiceSet.Has(v.ServiceName) { + rcs = append(rcs, v) + } + } + renderset.ChartInfos = rcs - return prod, nil -} + // create new renderset + if err := commonservice.CreateHelmRenderSet(renderset, log); err != nil { + log.Errorf("failed to create renderset, name %s, err: %s", renderset.Name, err) + return e.ErrUpdateEnv.AddErr(err) + } -func GetProductIngress(username, productName string, log *zap.SugaredLogger) ([]*ProductIngressInfo, error) { - productIngressInfos := make([]*ProductIngressInfo, 0) - products, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{Name: productName}) + productInfo.Render.Revision = renderset.Revision + err = commonrepo.NewProductColl().UpdateRender(renderset.EnvName, productInfo.ProductName, productInfo.Render) if err != nil { - log.Errorf("[%s] Collections.Product.List error: %v", username, err) - return productIngressInfos, e.ErrListEnvs.AddDesc(err.Error()) + log.Errorf("failed to update product render info, renderName: %s, err: %s", productInfo.Render.Name, err) + return e.ErrUpdateEnv.AddErr(err) } - for _, prod := range products { - productIngressInfo := new(ProductIngressInfo) - productIngressInfo.EnvName = prod.EnvName - ingressInfos := make([]*commonservice.IngressInfo, 0) - serviceGroups, _, err := ListGroups("", prod.EnvName, productName, 0, 0, log) - if err != nil { - log.Errorf("GetProductIngress GetProductAndKubeClient err:%v", err) - continue + go func() { + failedServices := sync.Map{} + wg := sync.WaitGroup{} + for service, revision := range deletedSvcRevision { + wg.Add(1) + go func(product *models.Product, serviceName string, revision int64) { + defer wg.Done() + templateSvc, err := commonrepo.NewServiceColl().Find(&commonrepo.ServiceFindOption{ServiceName: serviceName, Revision: revision, ProductName: product.ProductName}) + if err != nil { + failedServices.Store(serviceName, err.Error()) + return + } + log.Infof("uninstall release for service: %s", serviceName) + if errUninstall := UninstallService(helmClient, productInfo, templateSvc, false); errUninstall != nil { + errStr := fmt.Sprintf("helm uninstall service %s err: %s", serviceName, errUninstall) + failedServices.Store(serviceName, errStr) + log.Error(errStr) + } + }(productInfo, service, revision) + } + wg.Wait() + errList := make([]string, 0) + failedServices.Range(func(key, value interface{}) bool { + errList = append(errList, value.(string)) + return true + }) + // send err message to user + if len(errList) > 0 { + title := fmt.Sprintf("[%s] 的 [%s] 环境服务删除失败", productInfo.ProductName, productInfo.EnvName) + commonservice.SendErrorMessage(userName, title, requestID, errors.New(strings.Join(errList, "\n")), log) } - for _, serviceGroup := range serviceGroups { - if serviceGroup.Ingress != nil && len(serviceGroup.Ingress.HostInfo) > 0 { - ingressInfos = append(ingressInfos, serviceGroup.Ingress) + + if productInfo.ShareEnv.Enable && !productInfo.ShareEnv.IsBase { + err = EnsureGrayEnvConfig(ctx, productInfo, kclient, istioClient) + if err != nil { + log.Errorf("Failed to ensure gray env config: %s", err) } } - productIngressInfo.IngressInfos = ingressInfos - productIngressInfos = append(productIngressInfos, productIngressInfo) - } - return productIngressInfos, nil + }() + + return nil } -// ListRenderCharts 获取集成环境中的values.yaml的值 -func ListRenderCharts(productName, envName string, log *zap.SugaredLogger) ([]*template.RenderChart, error) { - renderSetOpt := &commonrepo.RenderSetFindOption{Name: productName} - if envName != "" { - opt := &commonrepo.ProductFindOptions{Name: productName, EnvName: envName} - productResp, err := commonrepo.NewProductColl().Find(opt) +func deleteK8sProductServices(productInfo *commonmodels.Product, serviceNames []string, log *zap.SugaredLogger) error { + for serviceGroupIndex, serviceGroup := range productInfo.Services { + var group []*commonmodels.ProductService + for _, service := range serviceGroup { + if !util.InStringArray(service.ServiceName, serviceNames) { + group = append(group, service) + } + } + err := commonrepo.NewProductColl().UpdateGroup(productInfo.EnvName, productInfo.ProductName, serviceGroupIndex, group) if err != nil { - log.Errorf("GetProduct envName:%s, productName:%s, err:%+v", envName, productName, err) - return nil, e.ErrListRenderSets.AddDesc(err.Error()) + log.Errorf("update product error: %v", err) + return err } - - renderSetName := productResp.Namespace - renderSetOpt = &commonrepo.RenderSetFindOption{Name: renderSetName, Revision: productResp.Render.Revision} } - - renderSet, err := commonrepo.NewRenderSetColl().Find(renderSetOpt) + rs, err := commonrepo.NewRenderSetColl().Find(&commonrepo.RenderSetFindOption{ + EnvName: productInfo.EnvName, + Name: productInfo.Namespace, + ProductTmpl: productInfo.ProductName, + }) if err != nil { - log.Errorf("find helm renderset[%s] error: %v", productName, err) - return nil, e.ErrListRenderSets.AddDesc(err.Error()) + log.Errorf("get renderSet error: %v", err) + return err } - return renderSet.ChartInfos, nil -} - -func GetHelmChartVersions(productName, envName string, log *zap.SugaredLogger) ([]*commonmodels.HelmVersions, error) { - var ( - helmVersions = make([]*commonmodels.HelmVersions, 0) - chartInfoMap = make(map[string]*template.RenderChart) - ) - opt := &commonrepo.ProductFindOptions{Name: productName, EnvName: envName} - prod, err := commonrepo.NewProductColl().Find(opt) + var updatedKVs []*templatemodels.RenderKV + for _, v := range rs.KVs { + var updatedServices []string + for _, service := range v.Services { + if util.InStringArray(service, serviceNames) { + updatedServices = append(updatedServices, service) + } + } + v.Services = updatedServices + updatedKVs = append(updatedKVs, v) + } + rs.KVs = updatedKVs + err = commonrepo.NewRenderSetColl().Update(rs) if err != nil { - log.Errorf("[EnvName:%s][Product:%s] Product.FindByOwner error: %v", envName, productName, err) - return nil, e.ErrGetEnv + log.Errorf("update renderSet error: %v", err) + return err } - prodTmpl, err := templaterepo.NewProductColl().Find(productName) + ctx := context.TODO() + kclient, err := kubeclient.GetKubeClient(config.HubServerAddress(), productInfo.ClusterID) if err != nil { - log.Errorf("[EnvName:%s][Product:%s] get product template error: %v", envName, productName, err) - return nil, e.ErrGetEnv + return fmt.Errorf("failed to get kube client: %s", err) } - //当前环境的renderset - renderSetName := prod.Namespace - renderSetOpt := &commonrepo.RenderSetFindOption{Name: renderSetName, Revision: prod.Render.Revision} - renderSet, err := commonrepo.NewRenderSetColl().Find(renderSetOpt) + restConfig, err := kubeclient.GetRESTConfig(config.HubServerAddress(), productInfo.ClusterID) if err != nil { - log.Errorf("find helm renderset[%s] error: %v", renderSetName, err) - return helmVersions, err + return fmt.Errorf("failed to get rest config: %s", err) } - for _, chartInfo := range renderSet.ChartInfos { - chartInfoMap[chartInfo.ServiceName] = chartInfo + + istioClient, err := versionedclient.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("failed to new istio client: %s", err) } - //当前环境内的服务信息 - prodServiceMap := prod.GetServiceMap() + for _, name := range serviceNames { + selector := labels.Set{setting.ProductLabel: productInfo.ProductName, setting.ServiceLabel: name}.AsSelector() - // all services - serviceListOpt := &commonrepo.ServiceListOption{ - ProductName: productName, - Type: setting.HelmDeployType, - } - for _, serviceGroup := range prodTmpl.Services { - for _, serviceName := range serviceGroup { - serviceListOpt.InServices = append(serviceListOpt.InServices, &template.ServiceInfo{ - Name: serviceName, - Owner: productName, - }) + err = EnsureDeleteZadigService(ctx, productInfo, selector, kclient, istioClient) + if err != nil { + // Only record and do not block subsequent traversals. + log.Errorf("Failed to delete Zadig service: %s", err) } - } - //当前项目内最新的服务信息 - latestServices, err := commonrepo.NewServiceColl().ListMaxRevisions(serviceListOpt) - if err != nil { - log.Errorf("find service revision list error: %v", err) - return helmVersions, err - } - for _, latestSvc := range latestServices { - if prodService, ok := prodServiceMap[latestSvc.ServiceName]; ok { - delete(prodServiceMap, latestSvc.ServiceName) - if latestSvc.Revision == prodService.Revision { - continue - } - helmVersion := &commonmodels.HelmVersions{ - ServiceName: latestSvc.ServiceName, - LatestVersion: latestSvc.HelmChart.Version, - LatestValuesYaml: latestSvc.HelmChart.ValuesYaml, - } - if chartInfo, ok := chartInfoMap[latestSvc.ServiceName]; ok { - helmVersion.CurrentVersion = chartInfo.ChartVersion - helmVersion.CurrentValuesYaml = chartInfo.ValuesYaml - } - helmVersions = append(helmVersions, helmVersion) - } else { // new service - helmVersion := &commonmodels.HelmVersions{ - ServiceName: latestSvc.ServiceName, - LatestVersion: latestSvc.HelmChart.Version, - LatestValuesYaml: latestSvc.HelmChart.ValuesYaml, - } - helmVersions = append(helmVersions, helmVersion) + err = commonservice.DeleteNamespacedResource(productInfo.Namespace, selector, productInfo.ClusterID, log) + if err != nil { + // Only record and do not block subsequent traversals. + log.Errorf("delete resource of service %s error:%v", name, err) } } - // deleted service - for _, prodService := range prodServiceMap { - helmVersion := &commonmodels.HelmVersions{ - ServiceName: prodService.ServiceName, - } - if chartInfo, ok := chartInfoMap[prodService.ServiceName]; ok { - helmVersion.CurrentVersion = chartInfo.ChartVersion - helmVersion.CurrentValuesYaml = chartInfo.ValuesYaml + if productInfo.ShareEnv.Enable && !productInfo.ShareEnv.IsBase { + err = EnsureGrayEnvConfig(ctx, productInfo, kclient, istioClient) + if err != nil { + log.Errorf("Failed to ensure gray env config: %s", err) + return fmt.Errorf("failed to ensure gray env config: %s", err) } - helmVersions = append(helmVersions, helmVersion) } - return helmVersions, nil + return nil } func GetEstimatedRenderCharts(productName, envName, serviceNameListStr string, log *zap.SugaredLogger) ([]*commonservice.RenderChartArg, error) { @@ -1585,7 +2518,7 @@ func GetEstimatedRenderCharts(productName, envName, serviceNameListStr string, l } // find renderchart info in env - renderChartInEnv, err := GetRenderCharts(productName, envName, serviceNameListStr, log) + renderChartInEnv, err := commonservice.GetRenderCharts(productName, envName, serviceNameListStr, log) if err != nil { log.Errorf("find render charts in env fail, env %s err %s", envName, err.Error()) return nil, e.ErrGetRenderSet.AddDesc("failed to get render charts in env") @@ -1605,7 +2538,7 @@ func GetEstimatedRenderCharts(productName, envName, serviceNameListStr string, l if _, ok := rcMap[serviceName]; ok { continue } - serviceOption.InServices = append(serviceOption.InServices, &template.ServiceInfo{ + serviceOption.InServices = append(serviceOption.InServices, &templatemodels.ServiceInfo{ Name: serviceName, Owner: productName, }) @@ -1633,7 +2566,7 @@ func GetEstimatedRenderCharts(productName, envName, serviceNameListStr string, l return ret, nil } -func createGroups(envName, user, requestID string, args *commonmodels.Product, eventStart int64, renderSet *commonmodels.RenderSet, kubeClient client.Client, log *zap.SugaredLogger) { +func createGroups(envName, user, requestID string, args *commonmodels.Product, eventStart int64, renderSet *commonmodels.RenderSet, informer informers.SharedInformerFactory, kubeClient client.Client, istioClient versionedclient.Interface, log *zap.SugaredLogger) { var err error defer func() { status := setting.ProductStatusSuccess @@ -1643,30 +2576,50 @@ func createGroups(envName, user, requestID string, args *commonmodels.Product, e errorMsg = err.Error() // 发送创建产品失败消息给用户 - title := fmt.Sprintf("创建 [%s] 的 [%s] 环境失败", args.ProductName, args.EnvName) + title := fmt.Sprintf("创建 [%s] 的 [%s] 环境失败:%s", args.ProductName, args.EnvName, errorMsg) commonservice.SendErrorMessage(user, title, requestID, err, log) } commonservice.LogProductStats(envName, setting.CreateProductEvent, args.ProductName, requestID, eventStart, log) if err := commonrepo.NewProductColl().UpdateStatus(envName, args.ProductName, status); err != nil { - log.Errorf("[%s][P:%s] Product.UpdateStatus error: %v", envName, args.ProductName, err) + log.Errorf("[%s][P:%s] Product.UpdateStatus error: %s", envName, args.ProductName, err) return } if err := commonrepo.NewProductColl().UpdateErrors(envName, args.ProductName, errorMsg); err != nil { - log.Errorf("[%s][P:%s] Product.UpdateErrors error: %v", envName, args.ProductName, err) + log.Errorf("[%s][P:%s] Product.UpdateErrors error: %s", envName, args.ProductName, err) return } }() + err = initEnvConfigSetAction(args.EnvName, args.Namespace, args.ProductName, user, args.EnvConfigs, false, kubeClient) + if err != nil { + args.Status = setting.ProductStatusFailed + log.Errorf("initEnvConfigSet error :%s", err) + return + } + for _, group := range args.Services { - err = envHandleFunc(getProjectType(args.ProductName), log).createGroup(envName, args.ProductName, user, group, renderSet, kubeClient) + err = envHandleFunc(getProjectType(args.ProductName), log).createGroup(envName, args.ProductName, user, group, renderSet, informer, kubeClient) if err != nil { args.Status = setting.ProductStatusFailed log.Errorf("createGroup error :%+v", err) return } } + + // If the user does not enable environment sharing, end. Otherwise, continue to perform environment sharing operations. + if !args.ShareEnv.Enable { + return + } + + // Note: Currently, only sub-environments can be created, but baseline environments cannot be created. + err = EnsureGrayEnvConfig(context.TODO(), args, kubeClient, istioClient) + if err != nil { + args.Status = setting.ProductStatusFailed + log.Errorf("Failed to ensure environment sharing in env %s of product %s: %s", args.EnvName, args.ProductName, err) + return + } } func getProjectType(productName string) string { @@ -1676,6 +2629,10 @@ func getProjectType(productName string) string { return projectType } + if projectInfo.ProductFeature.DeployType == setting.HelmDeployType { + return setting.HelmDeployType + } + if projectInfo.ProductFeature.DeployType == setting.K8SDeployType && projectInfo.ProductFeature.BasicFacility == setting.BasicFacilityK8S { return projectType } @@ -1686,153 +2643,10 @@ func getProjectType(productName string) string { return projectType } -// createGroup create or update services in service group -//func createGroup(envName, productName, username string, group []*commonmodels.ProductService, renderSet *commonmodels.RenderSet, kubeClient client.Client, log *zap.SugaredLogger) error { -// log.Infof("[Namespace:%s][Product:%s] createGroup", envName, productName) -// updatableServiceNameList := make([]string, 0) -// -// // 异步创建无依赖的服务 -// errList := &multierror.Error{ -// ErrorFormat: func(es []error) string { -// points := make([]string, len(es)) -// for i, err := range es { -// points[i] = fmt.Sprintf("%v", err) -// } -// -// return strings.Join(points, "\n") -// }, -// } -// -// opt := &commonrepo.ProductFindOptions{Name: productName, EnvName: envName} -// prod, err := commonrepo.NewProductColl().Find(opt) -// if err != nil { -// errList = multierror.Append(errList, err) -// } -// -// var wg sync.WaitGroup -// var lock sync.Mutex -// var resources []*unstructured.Unstructured -// -// for i := range group { -// if group[i].Type == setting.K8SDeployType { -// // 只有在service有Pod的时候，才需要等待pod running或者等待pod succeed -// // 比如在group中，如果service下仅有configmap/service/ingress这些yaml的时候，不需要waitServicesRunning -// wg.Add(1) -// updatableServiceNameList = append(updatableServiceNameList, group[i].ServiceName) -// go func(svc *commonmodels.ProductService) { -// defer wg.Done() -// items, err := upsertService(false, prod, svc, nil, renderSet, kubeClient, log) -// if err != nil { -// lock.Lock() -// switch e := err.(type) { -// case *multierror.Error: -// errList = multierror.Append(errList, e.Errors...) -// default: -// errList = multierror.Append(errList, e) -// } -// lock.Unlock() -// } -// -// // concurrent array append -// lock.Lock() -// resources = append(resources, items...) -// lock.Unlock() -// }(group[i]) -// } else if group[i].Type == setting.PMDeployType { -// //更新非k8s服务 -// if len(group[i].EnvConfigs) > 0 { -// serviceTempl, err := commonservice.GetServiceTemplate(group[i].ServiceName, setting.PMDeployType, productName, setting.ProductStatusDeleting, group[i].Revision, log) -// if err != nil { -// errList = multierror.Append(errList, err) -// } -// if serviceTempl != nil { -// oldEnvConfigs := serviceTempl.EnvConfigs -// for _, currentEnvConfig := range group[i].EnvConfigs { -// envConfig := &commonmodels.EnvConfig{ -// EnvName: currentEnvConfig.EnvName, -// HostIDs: currentEnvConfig.HostIDs, -// } -// oldEnvConfigs = append(oldEnvConfigs, envConfig) -// } -// -// args := &commonservice.ServiceTmplBuildObject{ -// ServiceTmplObject: &commonservice.ServiceTmplObject{ -// ProductName: serviceTempl.ProductName, -// ServiceName: serviceTempl.ServiceName, -// Visibility: serviceTempl.Visibility, -// Revision: serviceTempl.Revision, -// Type: serviceTempl.Type, -// Username: username, -// HealthChecks: serviceTempl.HealthChecks, -// EnvConfigs: oldEnvConfigs, -// EnvStatuses: []*commonmodels.EnvStatus{}, -// From: "createEnv", -// }, -// Build: &commonmodels.Build{Name: serviceTempl.BuildName}, -// } -// -// if err := commonservice.UpdatePmServiceTemplate(username, args, log); err != nil { -// errList = multierror.Append(errList, err) -// } -// } -// } -// var latestRevision int64 = group[i].Revision -// // 获取最新版本的服务 -// if latestServiceTempl, _ := commonservice.GetServiceTemplate(group[i].ServiceName, setting.PMDeployType, productName, setting.ProductStatusDeleting, 0, log); latestServiceTempl != nil { -// latestRevision = latestServiceTempl.Revision -// } -// // 更新环境 -// if latestRevision > group[i].Revision { -// // 更新产品服务 -// for _, serviceGroup := range prod.Services { -// for j, service := range serviceGroup { -// if service.ServiceName == group[i].ServiceName && service.Type == setting.PMDeployType { -// serviceGroup[j].Revision = latestRevision -// } -// } -// } -// if err := commonrepo.NewProductColl().Update(prod); err != nil { -// log.Errorf("[%s][%s] Product.Update error: %v", envName, productName, err) -// errList = multierror.Append(errList, err) -// } -// } -// if _, err = commonservice.CreateServiceTask(&commonmodels.ServiceTaskArgs{ -// ProductName: productName, -// ServiceName: group[i].ServiceName, -// Revision: latestRevision, -// EnvNames: []string{envName}, -// ServiceTaskCreator: username, -// }, log); err != nil { -// errList = multierror.Append(errList, err) -// } -// } -// } -// -// wg.Wait() -// -// // 如果创建依赖服务组有返回错误, 停止等待 -// if err := errList.ErrorOrNil(); err != nil { -// return err -// } -// -// if err := waitResourceRunning(kubeClient, prod.Namespace, resources, config.ServiceStartTimeout(), log); err != nil { -// log.Errorf( -// "service group %s/%+v doesn't start in %d seconds: %v", -// prod.Namespace, -// updatableServiceNameList, config.ServiceStartTimeout(), err) -// -// err = e.ErrUpdateEnv.AddErr( -// errors.Errorf(e.StartPodTimeout+"\n %s", "["+strings.Join(updatableServiceNameList, "], [")+"]")) -// return err -// } -// -// return nil -//} - // upsertService 创建或者更新服务, 更新服务之前先创建服务需要的配置 func upsertService(isUpdate bool, env *commonmodels.Product, service *commonmodels.ProductService, prevSvc *commonmodels.ProductService, - renderSet *commonmodels.RenderSet, kubeClient client.Client, log *zap.SugaredLogger, + renderSet *commonmodels.RenderSet, informer informers.SharedInformerFactory, kubeClient client.Client, istioClient versionedclient.Interface, log *zap.SugaredLogger, ) ([]*unstructured.Unstructured, error) { errList := &multierror.Error{ ErrorFormat: func(es []error) string { @@ -1903,12 +2717,9 @@ func upsertService(isUpdate bool, env *commonmodels.Product, switch u.GetKind() { case setting.Ingress: ls := kube.MergeLabels(labels, u.GetLabels()) - as := applySystemIngressTimeouts(u.GetAnnotations()) - as = applySystemIngressClass(as) u.SetNamespace(namespace) u.SetLabels(ls) - u.SetAnnotations(as) err = updater.CreateOrPatchUnstructured(u, kubeClient) if err != nil { @@ -1937,31 +2748,60 @@ func upsertService(isUpdate bool, env *commonmodels.Product, continue } + if istioClient != nil { + err = EnsureUpdateZadigService(context.TODO(), env, u.GetName(), kubeClient, istioClient) + if err != nil { + log.Errorf("Failed to update Zadig service %s for env %s of product %s: %s", u.GetName(), env.EnvName, env.ProductName, err) + errList = multierror.Append(errList, err) + continue + } + } case setting.Deployment, setting.StatefulSet: + // compatibility flag, We add a match label in spec.selector field pre 1.10. + needSelectorLabel := false + u.SetNamespace(namespace) - u.SetAPIVersion(setting.APIVersionAppsV1) u.SetLabels(kube.MergeLabels(labels, u.GetLabels())) - podLabels, _, _ := unstructured.NestedStringMap(u.Object, "spec", "template", "metadata", "labels") - err := unstructured.SetNestedStringMap(u.Object, kube.MergeLabels(labels, podLabels), "spec", "template", "metadata", "labels") + switch u.GetKind() { + case setting.Deployment: + needSelectorLabel = deploymentSelectorLabelExists(u.GetName(), namespace, informer, log) + case setting.StatefulSet: + needSelectorLabel = statefulsetSelectorLabelExists(u.GetName(), namespace, informer, log) + } + + podLabels, _, err := unstructured.NestedStringMap(u.Object, "spec", "template", "metadata", "labels") + if err != nil { + podLabels = nil + } + err = unstructured.SetNestedStringMap(u.Object, kube.MergeLabels(labels, podLabels), "spec", "template", "metadata", "labels") if err != nil { - // should not have happened - panic(err) + log.Errorf("merge label failed err:%s", err) + u.Object = setFieldValueIsNotExist(u.Object, kube.MergeLabels(labels, podLabels), "spec", "template", "metadata", "labels") } - podAnnotations, _, _ := unstructured.NestedStringMap(u.Object, "spec", "template", "metadata", "annotations") + podAnnotations, _, err := unstructured.NestedStringMap(u.Object, "spec", "template", "metadata", "annotations") + if err != nil { + podAnnotations = nil + } err = unstructured.SetNestedStringMap(u.Object, applyUpdatedAnnotations(podAnnotations), "spec", "template", "metadata", "annotations") if err != nil { - // should not have happened - panic(err) + log.Errorf("merge annotation failed err:%s", err) + u.Object = setFieldValueIsNotExist(u.Object, applyUpdatedAnnotations(podAnnotations), "spec", "template", "metadata", "annotations") } - // Inject selector: s-product and s-service - selector, _, _ := unstructured.NestedStringMap(u.Object, "spec", "selector", "matchLabels") - err = unstructured.SetNestedStringMap(u.Object, kube.MergeLabels(labels, selector), "spec", "selector", "matchLabels") - if err != nil { - // should not have happened - panic(err) + if needSelectorLabel { + // Inject selector: s-product and s-service + selector, _, err := unstructured.NestedStringMap(u.Object, "spec", "selector", "matchLabels") + if err != nil { + selector = nil + } + + err = unstructured.SetNestedStringMap(u.Object, kube.MergeLabels(labels, selector), "spec", "selector", "matchLabels") + if err != nil { + log.Errorf("merge selector failed err:%s", err) + u.Object = setFieldValueIsNotExist(u.Object, kube.MergeLabels(labels, selector), "spec", "selector", "matchLabels") + } } jsonData, err := u.MarshalJSON() @@ -1979,8 +2819,6 @@ func upsertService(isUpdate bool, env *commonmodels.Product, switch res := obj.(type) { case *appsv1.Deployment: - // Inject resource request and limit - applySystemResourceRequirements(&res.Spec.Template.Spec) // Inject imagePullSecrets if qn-registry-secret is not set applySystemImagePullSecrets(&res.Spec.Template.Spec) @@ -1991,8 +2829,6 @@ func upsertService(isUpdate bool, env *commonmodels.Product, continue } case *appsv1.StatefulSet: - // Inject resource request and limit - applySystemResourceRequirements(&res.Spec.Template.Spec) // Inject imagePullSecrets if qn-registry-secret is not set applySystemImagePullSecrets(&res.Spec.Template.Spec) @@ -2003,7 +2839,7 @@ func upsertService(isUpdate bool, env *commonmodels.Product, continue } default: - errList = multierror.Append(errList, fmt.Errorf("object is not a appsv1.Deployment ort appsv1.StatefulSet")) + errList = multierror.Append(errList, fmt.Errorf("object is not a appsv1.Deployment or appsv1.StatefulSet")) continue } @@ -2025,7 +2861,6 @@ func upsertService(isUpdate bool, env *commonmodels.Product, obj.ObjectMeta.Labels = kube.MergeLabels(labels, obj.ObjectMeta.Labels) obj.Spec.Template.ObjectMeta.Labels = kube.MergeLabels(labels, obj.Spec.Template.ObjectMeta.Labels) - applySystemResourceRequirements(&obj.Spec.Template.Spec) // Inject imagePullSecrets if qn-registry-secret is not set applySystemImagePullSecrets(&obj.Spec.Template.Spec) @@ -2060,7 +2895,6 @@ func upsertService(isUpdate bool, env *commonmodels.Product, obj.Spec.JobTemplate.ObjectMeta.Labels = kube.MergeLabels(labels, obj.Spec.JobTemplate.ObjectMeta.Labels) obj.Spec.JobTemplate.Spec.Template.ObjectMeta.Labels = kube.MergeLabels(labels, obj.Spec.JobTemplate.Spec.Template.ObjectMeta.Labels) - applySystemResourceRequirements(&obj.Spec.JobTemplate.Spec.Template.Spec) // Inject imagePullSecrets if qn-registry-secret is not set applySystemImagePullSecrets(&obj.Spec.JobTemplate.Spec.Template.Spec) @@ -2105,7 +2939,12 @@ func removeOldResources( kubeClient client.Client, log *zap.SugaredLogger, ) error { - opt := &commonrepo.RenderSetFindOption{Name: oldService.Render.Name, Revision: oldService.Render.Revision} + opt := &commonrepo.RenderSetFindOption{ + Name: oldService.Render.Name, + Revision: oldService.Render.Revision, + EnvName: env.EnvName, + ProductTmpl: env.ProductName, + } resp, err := commonrepo.NewRenderSetColl().Find(opt) if err != nil { log.Errorf("find renderset[%s/%d] error: %v", opt.Name, opt.Revision, err) @@ -2137,6 +2976,7 @@ func removeOldResources( for name, item := range oldItemsMap { _, exists := itemsMap[name] + item.SetNamespace(env.Namespace) if !exists { if err = updater.DeleteUnstructured(item, kubeClient); err != nil { log.Errorf( @@ -2250,7 +3090,8 @@ func waitResourceRunning( }) } -func preCreateProduct(envName string, args *commonmodels.Product, kubeClient client.Client, log *zap.SugaredLogger) error { +func preCreateProduct(envName string, args *commonmodels.Product, kubeClient client.Client, + log *zap.SugaredLogger) error { var ( productTemplateName = args.ProductName renderSetName = commonservice.GetProductEnvNamespace(envName, args.ProductName, args.Namespace) @@ -2295,7 +3136,7 @@ func preCreateProduct(envName string, args *commonmodels.Product, kubeClient cli args.Vars = nil - var productTmpl *template.Product + var productTmpl *templatemodels.Product // 查询产品模板 productTmpl, err = templaterepo.NewProductColl().Find(productTemplateName) if err != nil { @@ -2331,12 +3172,12 @@ func preCreateProduct(envName string, args *commonmodels.Product, kubeClient cli args.Render = tmpRenderInfo if preCreateNSAndSecret(productTmpl.ProductFeature) { - return ensureKubeEnv(args.Namespace, kubeClient, log) + return ensureKubeEnv(args.Namespace, args.RegistryID, args.ShareEnv.Enable, kubeClient, log) } return nil } -func preCreateNSAndSecret(productFeature *template.ProductFeature) bool { +func preCreateNSAndSecret(productFeature *templatemodels.ProductFeature) bool { if productFeature == nil { return true } @@ -2359,40 +3200,6 @@ func getPredefinedClusterLabels(product, service, envName string) map[string]str return labels } -func applySystemIngressTimeouts(labels map[string]string) map[string]string { - if labels == nil { - labels = make(map[string]string) - } - - if _, ok := labels[setting.IngressProxyConnectTimeoutLabel]; !ok { - labels[setting.IngressProxyConnectTimeoutLabel] = "300" - } - - if _, ok := labels[setting.IngressProxySendTimeoutLabel]; !ok { - labels[setting.IngressProxySendTimeoutLabel] = "300" - } - - if _, ok := labels[setting.IngressProxyReadTimeoutLabel]; !ok { - labels[setting.IngressProxyReadTimeoutLabel] = "300" - } - - return labels -} - -func applySystemIngressClass(labels map[string]string) map[string]string { - if labels == nil { - labels = make(map[string]string) - } - - if config.DefaultIngressClass() != "" { - if _, ok := labels[setting.IngressClassLabel]; !ok { - labels[setting.IngressClassLabel] = config.DefaultIngressClass() - } - } - - return labels -} - func applyUpdatedAnnotations(annotations map[string]string) map[string]string { if annotations == nil { annotations = make(map[string]string) @@ -2402,35 +3209,6 @@ func applyUpdatedAnnotations(annotations map[string]string) map[string]string { return annotations } -func applySystemResourceRequirements(podSpec *corev1.PodSpec) { - for i, container := range podSpec.Containers { - - if container.Resources.Limits == nil { - podSpec.Containers[i].Resources.Limits = corev1.ResourceList{} - } - - if container.Resources.Limits.Cpu().String() == "0" { - podSpec.Containers[i].Resources.Limits[corev1.ResourceCPU] = resource.MustParse("500m") - } - - if container.Resources.Limits.Memory().String() == "0" { - podSpec.Containers[i].Resources.Limits[corev1.ResourceMemory] = resource.MustParse("200Mi") - } - - if container.Resources.Requests == nil { - podSpec.Containers[i].Resources.Requests = corev1.ResourceList{} - } - - if container.Resources.Requests.Cpu().String() == "0" { - podSpec.Containers[i].Resources.Requests[corev1.ResourceCPU] = resource.MustParse("10m") - } - - if container.Resources.Requests.Memory().String() == "0" { - podSpec.Containers[i].Resources.Requests[corev1.ResourceMemory] = resource.MustParse("100Mi") - } - } -} - func applySystemImagePullSecrets(podSpec *corev1.PodSpec) { for _, secret := range podSpec.ImagePullSecrets { if secret.Name == setting.DefaultImagePullSecret { @@ -2443,15 +3221,15 @@ func applySystemImagePullSecrets(podSpec *corev1.PodSpec) { }) } -func ensureKubeEnv(namespace string, kubeClient client.Client, log *zap.SugaredLogger) error { - err := kube.CreateNamespace(namespace, kubeClient) +func ensureKubeEnv(namespace, registryId string, enableShare bool, kubeClient client.Client, log *zap.SugaredLogger) error { + err := kube.CreateNamespace(namespace, enableShare, kubeClient) if err != nil { log.Errorf("[%s] get or create namespace error: %v", namespace, err) - return e.ErrCreateNamspace.AddDesc(e.SetNamespaceErrMsg) + return e.ErrCreateNamspace.AddDesc(err.Error()) } // 创建默认的镜像仓库secret - if err := commonservice.EnsureDefaultRegistrySecret(namespace, kubeClient, log); err != nil { + if err := commonservice.EnsureDefaultRegistrySecret(namespace, registryId, kubeClient, log); err != nil { log.Errorf("[%s] get or create namespace error: %v", namespace, err) return e.ErrCreateSecret.AddDesc(e.CreateDefaultRegistryErrMsg) } @@ -2459,38 +3237,52 @@ func ensureKubeEnv(namespace string, kubeClient client.Client, log *zap.SugaredL return nil } -func FindHelmRenderSet(productName, renderName string, log *zap.SugaredLogger) (*commonmodels.RenderSet, error) { +func FindHelmRenderSet(productName, renderName, envName string, log *zap.SugaredLogger) (*commonmodels.RenderSet, error) { resp := &commonmodels.RenderSet{ProductTmpl: productName} var err error if renderName != "" { - opt := &commonrepo.RenderSetFindOption{Name: renderName} + opt := &commonrepo.RenderSetFindOption{ + Name: renderName, + ProductTmpl: productName, + EnvName: envName, + } resp, err = commonrepo.NewRenderSetColl().Find(opt) if err != nil { log.Errorf("find helm renderset[%s] error: %v", renderName, err) return resp, err } } - if renderName != "" && resp.ProductTmpl != productName { - log.Errorf("helm renderset[%s] not match product[%s]", renderName, productName) - return resp, fmt.Errorf("helm renderset[%s] not match product[%s]", renderName, productName) - } return resp, nil } -func installOrUpgradeHelmChart(namespace string, renderChart *template.RenderChart, serviceObj *commonmodels.Service, timeout time.Duration, helmClient helmclient.Client) error { - mergedValuesYaml, err := helmtool.MergeOverrideValues(renderChart.ValuesYaml, renderChart.GetOverrideYaml(), renderChart.OverrideValues) +func buildInstallParam(namespace, envName, defaultValues string, renderChart *templatemodels.RenderChart, serviceObj *commonmodels.Service) (*ReleaseInstallParam, error) { + mergedValues, err := helmtool.MergeOverrideValues(renderChart.ValuesYaml, defaultValues, renderChart.GetOverrideYaml(), renderChart.OverrideValues) if err != nil { - err = errors.WithMessagef(err, "failed to merge override yaml %s and values %s", renderChart.GetOverrideYaml(), renderChart.OverrideValues) - return err + return nil, fmt.Errorf("failed to merge override yaml %s and values %s, err: %s", renderChart.GetOverrideYaml(), renderChart.OverrideValues, err) + } + ret := &ReleaseInstallParam{ + ProductName: serviceObj.ProductName, + Namespace: namespace, + ReleaseName: util.GeneReleaseName(serviceObj.GetReleaseNaming(), serviceObj.ProductName, namespace, envName, serviceObj.ServiceName), + MergedValues: mergedValues, + RenderChart: renderChart, + serviceObj: serviceObj, } - return installOrUpgradeHelmChartWithValues(namespace, mergedValuesYaml, renderChart, serviceObj, timeout, helmClient) + return ret, nil } -func installOrUpgradeHelmChartWithValues(namespace, valuesYaml string, renderChart *template.RenderChart, serviceObj *commonmodels.Service, timeout time.Duration, helmClient helmclient.Client) error { - base := config.LocalServicePath(serviceObj.ProductName, serviceObj.ServiceName) - if err := commonservice.PreLoadServiceManifests(base, serviceObj); err != nil { - log.Errorf("Failed to load manifest for service %s in project %s, err: %s", serviceObj.ServiceName, serviceObj.ProductName, err) - return err +func installOrUpgradeHelmChartWithValues(param *ReleaseInstallParam, isRetry bool, helmClient *helmtool.HelmClient) error { + namespace, valuesYaml, renderChart, serviceObj := param.Namespace, param.MergedValues, param.RenderChart, param.serviceObj + base := config.LocalServicePathWithRevision(serviceObj.ProductName, serviceObj.ServiceName, serviceObj.Revision) + if err := commonservice.PreloadServiceManifestsByRevision(base, serviceObj); err != nil { + log.Warnf("failed to get chart of revision: %d for service: %s, use latest version", + serviceObj.Revision, serviceObj.ServiceName) + // use the latest version when it fails to download the specific version + base = config.LocalServicePath(serviceObj.ProductName, serviceObj.ServiceName) + if err = commonservice.PreLoadServiceManifests(base, serviceObj); err != nil { + log.Errorf("failed to load chart info for service %v", serviceObj.ServiceName) + return fmt.Errorf("failed to load chart info for service %s", serviceObj.ServiceName) + } } chartFullPath := filepath.Join(base, serviceObj.ServiceName) @@ -2501,98 +3293,100 @@ func installOrUpgradeHelmChartWithValues(namespace, valuesYaml string, renderCha } chartSpec := &helmclient.ChartSpec{ - ReleaseName: util.GeneHelmReleaseName(namespace, serviceObj.ServiceName), - ChartName: chartPath, - Namespace: namespace, - Version: renderChart.ChartVersion, - ValuesYaml: valuesYaml, - UpgradeCRDs: true, + ReleaseName: param.ReleaseName, + ChartName: chartPath, + Namespace: namespace, + Version: renderChart.ChartVersion, + ValuesYaml: valuesYaml, + UpgradeCRDs: true, + CleanupOnFail: true, + MaxHistory: 10, + DryRun: param.DryRun, + } + if isRetry { + chartSpec.Replace = true + } + + // If the target environment is a shared environment and a sub env, we need to clear the deployed K8s Service. + ctx := context.TODO() + if !chartSpec.DryRun { + err = EnsureDeletePreCreatedServices(ctx, param.ProductName, param.Namespace, chartSpec, helmClient) + if err != nil { + return fmt.Errorf("failed to ensure deleting pre-created K8s Services for product %q in namespace %q: %s", param.ProductName, param.Namespace, err) + } } - if timeout > 0 { - chartSpec.Wait = true - chartSpec.Timeout = timeout + + helmClient, err = helmClient.Clone() + if err != nil { + return fmt.Errorf("failed to clone helm client: %s", err) } - if _, err = helmClient.InstallOrUpgradeChart(context.TODO(), chartSpec); err != nil { - return err + var release *release.Release + release, err = helmClient.InstallOrUpgradeChart(ctx, chartSpec) + if err != nil { + err = errors.WithMessagef( + err, + "failed to install or upgrade helm chart %s/%s", + namespace, serviceObj.ServiceName) + } else { + if !chartSpec.DryRun { + err = EnsureZadigServiceByManifest(ctx, param.ProductName, param.Namespace, release.Manifest) + if err != nil { + err = errors.WithMessagef(err, "failed to ensure Zadig Service %s", err) + } + } } - return nil + + return err } -func installProductHelmCharts(user, envName, requestID string, args *commonmodels.Product, eventStart int64, helmClient helmclient.Client, log *zap.SugaredLogger) { +func installProductHelmCharts(user, envName, requestID string, args *commonmodels.Product, renderset *commonmodels.RenderSet, eventStart int64, helmClient *helmtool.HelmClient, + kclient client.Client, istioClient versionedclient.Interface, log *zap.SugaredLogger) { var ( err error - wg sync.WaitGroup errList = &multierror.Error{} ) defer func() { - status := setting.ProductStatusSuccess - errorMsg := "" if err != nil { - status = setting.ProductStatusFailed - errorMsg = err.Error() - - // 发送创建产品失败消息给用户 title := fmt.Sprintf("创建 [%s] 的 [%s] 环境失败", args.ProductName, args.EnvName) commonservice.SendErrorMessage(user, title, requestID, err, log) } commonservice.LogProductStats(envName, setting.CreateProductEvent, args.ProductName, requestID, eventStart, log) - if err := commonrepo.NewProductColl().UpdateStatus(envName, args.ProductName, status); err != nil { - log.Errorf("[%s][P:%s] Product.UpdateStatus error: %v", envName, args.ProductName, err) - return - } - if err := commonrepo.NewProductColl().UpdateErrors(envName, args.ProductName, errorMsg); err != nil { - log.Errorf("[%s][P:%s] Product.UpdateErrors error: %v", envName, args.ProductName, err) + status := setting.ProductStatusSuccess + if err = commonrepo.NewProductColl().UpdateStatusAndError(envName, args.ProductName, status, ""); err != nil { + log.Errorf("[%s][P:%s] Product.UpdateStatusAndError error: %v", envName, args.ProductName, err) return } }() - chartInfoMap := make(map[string]*template.RenderChart) + + chartInfoMap := make(map[string]*templatemodels.RenderChart) for _, renderChart := range args.ChartInfos { chartInfoMap[renderChart.ServiceName] = renderChart } - for _, serviceGroups := range args.Services { - for _, svc := range serviceGroups { - renderChart, ok := chartInfoMap[svc.ServiceName] - if !ok { - continue - } - wg.Add(1) - go func(service *commonmodels.ProductService) { - defer wg.Done() - - // 获取服务详情 - opt := &commonrepo.ServiceFindOption{ - ServiceName: service.ServiceName, - Type: service.Type, - Revision: service.Revision, - ProductName: args.ProductName, - ExcludeStatus: setting.ProductStatusDeleting, - } - serviceObj, err := commonrepo.NewServiceColl().Find(opt) - if err != nil { - return - } + err = proceedHelmRelease(args.ProductName, args.EnvName, args, renderset, helmClient, nil, log) + if err != nil { + log.Errorf("error occurred when installing services in env: %s/%s, err: %s ", args.ProductName, envName, err) + errList = multierror.Append(errList, err) + } - err = installOrUpgradeHelmChart(args.Namespace, renderChart, serviceObj, Timeout*time.Second*10, helmClient) - if err != nil { - errList = multierror.Append(errList, err) - return - } - }(svc) + // Note: For the sub env, try to supplement information relevant to the base env. + if args.ShareEnv.Enable && !args.ShareEnv.IsBase { + shareEnvErr := EnsureGrayEnvConfig(context.TODO(), args, kclient, istioClient) + if shareEnvErr != nil { + errList = multierror.Append(errList, shareEnvErr) } } - wg.Wait() + err = errList.ErrorOrNil() } func setServiceRender(args *commonmodels.Product) { for _, serviceGroup := range args.Services { for _, service := range serviceGroup { - // 当service type是k8s时才需要渲染信息 if service.Type == setting.K8SDeployType || service.Type == setting.HelmDeployType { service.Render = args.Render } @@ -2649,58 +3443,64 @@ func getUpdatedProductServices(updateProduct *commonmodels.Product, serviceRevis return updatedAllServices } -func updateProductGroup(productName, envName, updateType string, productResp *commonmodels.Product, currentProductServices [][]*commonmodels.ProductService, overrideCharts []*commonservice.RenderChartArg, log *zap.SugaredLogger) error { - var ( - renderChartMap = make(map[string]*template.RenderChart) - productServiceMap = make(map[string]*commonmodels.ProductService) - productTemplServiceMap = make(map[string]*commonmodels.ProductService) - ) - restConfig, err := kube.GetRESTConfig(productResp.ClusterID) - if err != nil { - return e.ErrUpdateEnv.AddErr(err) +func batchExecutorWithRetry(retryCount uint64, interval time.Duration, serviceList []*commonmodels.Service, handler intervalExecutorHandler, log *zap.SugaredLogger) []error { + bo := backoff.NewConstantBackOff(time.Second * 3) + retryBo := backoff.WithMaxRetries(bo, retryCount) + errList := make([]error, 0) + isRetry := false + _ = backoff.Retry(func() error { + failedServices := make([]*commonmodels.Service, 0) + errList = batchExecutor(interval, serviceList, &failedServices, isRetry, handler, log) + if len(errList) == 0 { + return nil + } + log.Infof("%d services waiting to retry", len(failedServices)) + serviceList = failedServices + isRetry = true + return fmt.Errorf("%d services apply failed", len(errList)) + }, retryBo) + return errList +} + +func batchExecutor(interval time.Duration, serviceList []*commonmodels.Service, failedServices *[]*commonmodels.Service, isRetry bool, handler intervalExecutorHandler, log *zap.SugaredLogger) []error { + if len(serviceList) == 0 { + return nil + } + errList := make([]error, 0) + for _, data := range serviceList { + err := handler(data, isRetry, log) + if err != nil { + errList = append(errList, err) + *failedServices = append(*failedServices, data) + log.Errorf("service:%s apply failed, err %s", data.ServiceName, err) + } + time.Sleep(interval) } + return errList +} - helmClient, err := helmtool.NewClientFromRestConf(restConfig, productResp.Namespace) +func updateProductGroup(username, productName, envName string, productResp *commonmodels.Product, + overrideCharts []*commonservice.RenderChartArg, deletedSvcRevision map[string]int64, log *zap.SugaredLogger) error { + + helmClient, err := helmtool.NewClientFromNamespace(productResp.ClusterID, productResp.Namespace) if err != nil { return e.ErrUpdateEnv.AddErr(err) } - for _, serviceGroup := range currentProductServices { - for _, service := range serviceGroup { - productServiceMap[service.ServiceName] = service - } - } - - for _, serviceGroup := range productResp.Services { - for _, service := range serviceGroup { - productTemplServiceMap[service.ServiceName] = service - } - } - - // 找到环境里面还存在但是服务编排里面已经删除的服务卸载掉 - for serviceName := range productServiceMap { - if _, isExist := productTemplServiceMap[serviceName]; !isExist { - go func(namespace, serviceName string) { - log.Infof("ready to uninstall release:%s", fmt.Sprintf("%s-%s", namespace, serviceName)) - if err = helmClient.UninstallRelease(&helmclient.ChartSpec{ - ReleaseName: util.GeneHelmReleaseName(namespace, serviceName), - Namespace: namespace, - Wait: true, - Force: true, - Timeout: Timeout * time.Second * 10, - }); err != nil { - log.Errorf("helm uninstall release %s err:%v", fmt.Sprintf("%s-%s", namespace, serviceName), err) - } - }(productResp.Namespace, serviceName) + // uninstall services + for serviceName, serviceRevision := range deletedSvcRevision { + if err = UninstallServiceByName(helmClient, serviceName, productResp, serviceRevision, true); err != nil { + log.Errorf("UninstallRelease err:%v", err) + return e.ErrUpdateEnv.AddErr(err) } } - //比较当前环境中的变量和系统默认的最新变量 - renderSet, err := diffRenderSet(productName, envName, updateType, productResp, overrideCharts, log) + renderSet, err := diffRenderSet(username, productName, envName, productResp, overrideCharts, log) if err != nil { return e.ErrUpdateEnv.AddDesc("对比环境中的value.yaml和系统默认的value.yaml失败") } + productResp.ChartInfos = renderSet.ChartInfos svcNameSet := sets.NewString() for _, singleChart := range overrideCharts { if singleChart.EnvName != envName { @@ -2709,161 +3509,148 @@ func updateProductGroup(productName, envName, updateType string, productResp *co svcNameSet.Insert(singleChart.ServiceName) } - for _, renderChart := range renderSet.ChartInfos { - renderChartMap[renderChart.ServiceName] = renderChart - } - - errList := new(multierror.Error) - for groupIndex, services := range productResp.Services { - var wg sync.WaitGroup - for _, svc := range services { - renderChart, ok := renderChartMap[svc.ServiceName] - if !ok { - continue - } - - // service is not in update list - if !svcNameSet.Has(svc.ServiceName) { - continue - } - - wg.Add(1) - go func(service *commonmodels.ProductService) { - defer wg.Done() - - opt := &commonrepo.ServiceFindOption{ - ServiceName: service.ServiceName, - Type: service.Type, - Revision: service.Revision, - ProductName: service.ProductName, - ExcludeStatus: setting.ProductStatusDeleting, - } - serviceObj, err := commonrepo.NewServiceColl().Find(opt) - if err != nil { - log.Errorf("Failed to find service with opt %+v, err: %s", opt, err) - errList = multierror.Append(errList, err) - return - } - - err = installOrUpgradeHelmChart(productResp.Namespace, renderChart, serviceObj, Timeout*time.Second*10, helmClient) - if err != nil { - errList = multierror.Append(errList, err) - return - } - }(svc) - } - - wg.Wait() - if err = commonrepo.NewProductColl().UpdateGroup(envName, productName, groupIndex, services); err != nil { - log.Errorf("Failed to update service group %d, err: %s", groupIndex, err) - errList = multierror.Append(errList, err) - } + filter := func(svc *commonmodels.ProductService) bool { + return svcNameSet.Has(svc.ServiceName) } productResp.Render.Revision = renderSet.Revision if err = commonrepo.NewProductColl().Update(productResp); err != nil { log.Errorf("Failed to update env, err: %s", err) - errList = multierror.Append(errList, err) + return err } - return errList.ErrorOrNil() -} - -// diffRenderSet 对比环境中的renderSet的值和服务的最新的renderSet的值 -func diffRenderSet(productName, envName, updateType string, productResp *commonmodels.Product, overrideCharts []*commonservice.RenderChartArg, log *zap.SugaredLogger) (*commonmodels.RenderSet, error) { - productTemp, err := templaterepo.NewProductColl().Find(productName) + err = proceedHelmRelease(productName, envName, productResp, renderSet, helmClient, filter, log) if err != nil { - log.Errorf("[ProductTmpl.find] err: %v", err) - return nil, err + log.Errorf("error occurred when upgrading services in env: %s/%s, err: %s ", productName, envName, err) + return err } - // 系统默认的变量 - latestRenderSet, err := commonrepo.NewRenderSetColl().Find(&commonrepo.RenderSetFindOption{Name: productName}) + + return nil +} + +// diffRenderSet get diff between renderset in product and product template +// generate a new renderset and insert into db +func diffRenderSet(username, productName, envName string, productResp *commonmodels.Product, overrideCharts []*commonservice.RenderChartArg, log *zap.SugaredLogger) (*commonmodels.RenderSet, error) { + // default renderset + latestRenderSet, err := commonrepo.NewRenderSetColl().Find(&commonrepo.RenderSetFindOption{Name: productName, IsDefault: true}) if err != nil { log.Errorf("[RenderSet.find] err: %v", err) return nil, err } - latestRenderSetMap := make(map[string]*template.RenderChart) + // chart infos in template + latestChartInfoMap := make(map[string]*templatemodels.RenderChart) for _, renderInfo := range latestRenderSet.ChartInfos { - latestRenderSetMap[renderInfo.ServiceName] = renderInfo + latestChartInfoMap[renderInfo.ServiceName] = renderInfo } + // chart infos from client renderChartArgMap := make(map[string]*commonservice.RenderChartArg) for _, singleArg := range overrideCharts { - if singleArg.EnvName != envName { - continue + if singleArg.EnvName == envName { + renderChartArgMap[singleArg.ServiceName] = singleArg } - renderChartArgMap[singleArg.ServiceName] = singleArg } - newChartInfos := make([]*template.RenderChart, 0) - renderSetName := productResp.Namespace - switch updateType { - case UpdateTypeSystem: - for _, serviceNameGroup := range productTemp.Services { - for _, serviceName := range serviceNameGroup { - if latestChartInfo, isExist := latestRenderSetMap[serviceName]; isExist { - newChartInfos = append(newChartInfos, latestChartInfo) - } - } - } - case UpdateTypeEnv: - renderSetOpt := &commonrepo.RenderSetFindOption{Name: renderSetName, Revision: productResp.Render.Revision} - currentEnvRenderSet, err := commonrepo.NewRenderSetColl().Find(renderSetOpt) - if err != nil { - log.Errorf("[RenderSet.find] err: %v", err) - return nil, err + renderSetOpt := &commonrepo.RenderSetFindOption{ + Name: productResp.Render.Name, + Revision: productResp.Render.Revision, + ProductTmpl: productName, + } + currentEnvRenderSet, err := commonrepo.NewRenderSetColl().Find(renderSetOpt) + if err != nil { + log.Errorf("[RenderSet.find] err: %v", err) + return nil, err + } + defaultValues, yamlData := currentEnvRenderSet.DefaultValues, currentEnvRenderSet.YamlData + + // chart infos in product + currentChartInfoMap := make(map[string]*templatemodels.RenderChart) + for _, renderInfo := range currentEnvRenderSet.ChartInfos { + currentChartInfoMap[renderInfo.ServiceName] = renderInfo + } + + opt := &commonrepo.ProductFindOptions{Name: productName, EnvName: envName} + productCur, err := commonrepo.NewProductColl().Find(opt) + if err != nil { + log.Errorf("GetProduct envName:%s, productName:%s, err:%s", envName, productName, err) + return nil, fmt.Errorf("GetProduct envName:%s, productName:%s, err:%s", envName, productName, err) + } + serviceMap := productCur.GetServiceMap() + serviceRespMap := productResp.GetServiceMap() + newChartInfos := make([]*templatemodels.RenderChart, 0) + + for serviceName, latestChartInfo := range latestChartInfoMap { + currentChartInfo, okC := currentChartInfoMap[serviceName] + renderArg, okR := renderChartArgMap[serviceName] + if !okR && !okC { + continue } - // 环境里面的变量 - currentEnvRenderSetMap := make(map[string]*template.RenderChart) - for _, renderInfo := range currentEnvRenderSet.ChartInfos { - currentEnvRenderSetMap[renderInfo.ServiceName] = renderInfo + // no need to update service revision in renderset.services + if !okR { + newChartInfos = append(newChartInfos, currentChartInfo) + continue } - tmpCurrentChartInfoMap := make(map[string]*template.RenderChart) - tmpLatestChartInfoMap := make(map[string]*template.RenderChart) - //过滤掉服务编排没有的服务，这部分不需要做diff - for _, serviceNameGroup := range productTemp.Services { - for _, serviceName := range serviceNameGroup { - if currentChartInfo, isExist := currentEnvRenderSetMap[serviceName]; isExist { - tmpCurrentChartInfoMap[serviceName] = currentChartInfo - } - if latestChartInfo, isExist := latestRenderSetMap[serviceName]; isExist { - tmpLatestChartInfoMap[serviceName] = latestChartInfo + serviceInfoResp := serviceRespMap[serviceName] + serviceInfoCur := serviceMap[serviceName] + imageRelatedKey := sets.NewString() + if serviceInfoResp != nil && serviceInfoCur != nil { + curEnvService, err := commonrepo.NewServiceColl().Find(&commonrepo.ServiceFindOption{ + ServiceName: serviceName, + ProductName: productName, + Type: setting.HelmDeployType, + Revision: serviceInfoCur.Revision, + }) + if err != nil { + log.Errorf("failed to query service, name %s, Revision %d,err %s", serviceName, serviceInfoCur.Revision, err) + return nil, fmt.Errorf("failed to query service, name %s,Revision %d,err %s", serviceName, serviceInfoCur.Revision, err) + } + L: + for _, curSvcContainers := range curEnvService.Containers { + if checkServiceImageUpdated(curSvcContainers, serviceInfoCur) { + for _, container := range serviceInfoResp.Containers { + if curSvcContainers.Name == container.Name && container.ImagePath != nil { + imageRelatedKey.Insert(container.ImagePath.Image, container.ImagePath.Repo, container.ImagePath.Tag) + continue L + } + } } } } - for serviceName, latestChartInfo := range tmpLatestChartInfoMap { - if currentChartInfo, ok := tmpCurrentChartInfoMap[serviceName]; ok { - //拿当前环境values.yaml的key的value去替换服务里面的values.yaml的相同的key的value - //TODO values.yaml should be the same in both currentChartInfo and latestChartInfo, since values.yaml can't be edited directly - newValuesYaml, err := overrideValues([]byte(currentChartInfo.ValuesYaml), []byte(latestChartInfo.ValuesYaml)) - if err != nil { - log.Errorf("Failed to override values, err: %s", err) - } else { - latestChartInfo.ValuesYaml = string(newValuesYaml) - } - - // user override value in cur environment - latestChartInfo.OverrideValues = currentChartInfo.OverrideValues - latestChartInfo.OverrideYaml = currentChartInfo.OverrideYaml - } - // user override value form request - if renderArg, ok := renderChartArgMap[serviceName]; ok { - renderArg.FillRenderChartModel(latestChartInfo, latestChartInfo.ChartVersion) + // use the variables in current product when updating services + if okC { + // use the value of the key of the current values.yaml to replace the value of the same key of the values.yaml in the service + newValuesYaml, err := overrideValues([]byte(currentChartInfo.ValuesYaml), []byte(latestChartInfo.ValuesYaml), imageRelatedKey) + if err != nil { + log.Errorf("Failed to override values for service %s, err: %s", serviceName, err) + } else { + latestChartInfo.ValuesYaml = string(newValuesYaml) } - newChartInfos = append(newChartInfos, latestChartInfo) + + // user override value in cur environment + latestChartInfo.OverrideValues = currentChartInfo.OverrideValues + latestChartInfo.OverrideYaml = currentChartInfo.OverrideYaml + } + + // user override value form request + if okR { + renderArg.FillRenderChartModel(latestChartInfo, latestChartInfo.ChartVersion) } + newChartInfos = append(newChartInfos, latestChartInfo) } if err = commonservice.CreateHelmRenderSet( &commonmodels.RenderSet{ - Name: renderSetName, - EnvName: envName, - ProductTmpl: productName, - ChartInfos: newChartInfos, + Name: productResp.Render.Name, + EnvName: envName, + ProductTmpl: productName, + ChartInfos: newChartInfos, + DefaultValues: defaultValues, + YamlData: yamlData, + UpdateBy: username, }, log, ); err != nil { @@ -2871,7 +3658,7 @@ func diffRenderSet(productName, envName, updateType string, productResp *commonm return nil, err } - renderSet, err := FindHelmRenderSet(productName, renderSetName, log) + renderSet, err := FindHelmRenderSet(productName, productResp.Render.Name, envName, log) if err != nil { log.Errorf("[RenderSet.find] err: %v", err) return nil, err @@ -2879,7 +3666,48 @@ func diffRenderSet(productName, envName, updateType string, productResp *commonm return renderSet, nil } -func overrideValues(currentValuesYaml, latestValuesYaml []byte) ([]byte, error) { +//checkServiceImageUpdated If the service does not do any mirroring iterations on the platform, the latest YAML is used when updating the environment +func checkServiceImageUpdated(curContainer *commonmodels.Container, serviceInfo *commonmodels.ProductService) bool { + for _, proContainer := range serviceInfo.Containers { + if curContainer.Name == proContainer.Name && curContainer.Image == proContainer.Image { + return false + } + } + return true +} + +// find related keys of service modules when container image changed in env since imported +func getImageChangeInvolvedKeys(prodService *commonmodels.ProductService, revisionSvc, latestSvc *commonmodels.Service) sets.String { + imageSet := sets.NewString() + latestContainer := make(map[string]*commonmodels.Container) + revisionSvcMap := make(map[string]*commonmodels.Container) + for _, c := range latestSvc.Containers { + latestContainer[c.Name] = c + } + for _, c := range revisionSvc.Containers { + revisionSvcMap[c.Name] = c + } + + for _, prodContainer := range prodService.Containers { + revisionSvcContainer, ok := revisionSvcMap[prodContainer.Name] + if !ok { + continue + } + if prodContainer.Image == revisionSvcContainer.Image { + continue + } + lc, ok := latestContainer[prodContainer.Name] + if !ok || lc.ImagePath == nil { + continue + } + imageSet.Insert(lc.ImagePath.Image, lc.ImagePath.Tag, lc.ImagePath.Repo) + } + return imageSet +} + +// for keys exist in both yaml, current values will override the latest values +// only for images +func overrideValues(currentValuesYaml, latestValuesYaml []byte, imageRelatedKey sets.String) ([]byte, error) { currentValuesMap := map[string]interface{}{} if err := yaml.Unmarshal(currentValuesYaml, ¤tValuesMap); err != nil { return nil, err @@ -2902,13 +3730,16 @@ func overrideValues(currentValuesYaml, latestValuesYaml []byte) ([]byte, error) replaceMap := make(map[string]interface{}) for key := range latestValuesFlatMap { + if !imageRelatedKey.Has(key) { + continue + } if currentValue, ok := currentValuesFlatMap[key]; ok { replaceMap[key] = currentValue } } if len(replaceMap) == 0 { - return nil, nil + return latestValuesYaml, nil } var replaceKV []string @@ -2923,62 +3754,180 @@ func overrideValues(currentValuesYaml, latestValuesYaml []byte) ([]byte, error) return yaml.Marshal(latestValuesMap) } -func updateProductVariable(productName, envName string, productResp *commonmodels.Product, log *zap.SugaredLogger) error { - restConfig, err := kube.GetRESTConfig(productResp.ClusterID) - if err != nil { - return e.ErrUpdateEnv.AddErr(err) +func dryRunInstallRelease(productResp *commonmodels.Product, renderset *commonmodels.RenderSet, helmClient *helmtool.HelmClient, log *zap.SugaredLogger) error { + productName, _ := productResp.ProductName, productResp.EnvName + renderChartMap := make(map[string]*templatemodels.RenderChart) + for _, renderChart := range productResp.ChartInfos { + renderChartMap[renderChart.ServiceName] = renderChart } - helmClient, err := helmtool.NewClientFromRestConf(restConfig, productResp.Namespace) - if err != nil { - return e.ErrUpdateEnv.AddErr(err) + handler := func(serviceObj *commonmodels.Service, log *zap.SugaredLogger) (err error) { + param, errBuildParam := buildInstallParam(productResp.Namespace, renderset.EnvName, renderset.DefaultValues, renderChartMap[serviceObj.ServiceName], serviceObj) + if errBuildParam != nil { + return errBuildParam + } + param.DryRun = true + err = installOrUpgradeHelmChartWithValues(param, false, helmClient) + return + } + + errList := new(multierror.Error) + var errLock sync.Mutex + appendErr := func(err error) { + errLock.Lock() + defer errLock.Unlock() + errList = multierror.Append(errList, err) + } + + var wg sync.WaitGroup + for _, groupServices := range productResp.Services { + serviceList := make([]*commonmodels.Service, 0) + for _, service := range groupServices { + if _, ok := renderChartMap[service.ServiceName]; !ok { + continue + } + opt := &commonrepo.ServiceFindOption{ + ServiceName: service.ServiceName, + Type: service.Type, + Revision: service.Revision, + ProductName: productName, + } + serviceObj, err := commonrepo.NewServiceColl().Find(opt) + if err != nil { + appendErr(fmt.Errorf("failed to find service %s, err %s", service.ServiceName, err.Error())) + continue + } + serviceList = append(serviceList, serviceObj) + } + + for _, svc := range serviceList { + wg.Add(1) + go func(service *models.Service) { + defer wg.Done() + err := handler(service, log) + if err != nil { + appendErr(fmt.Errorf("failed to dryRun install chart for service: %s, err: %s", service.ServiceName, err)) + } + }(svc) + } } + wg.Wait() + return errList.ErrorOrNil() +} - renderChartMap := make(map[string]*template.RenderChart) +func proceedHelmRelease(productName, envName string, productResp *commonmodels.Product, renderset *commonmodels.RenderSet, helmClient *helmtool.HelmClient, filter svcUpgradeFilter, log *zap.SugaredLogger) error { + renderChartMap := make(map[string]*templatemodels.RenderChart) for _, renderChart := range productResp.ChartInfos { renderChartMap[renderChart.ServiceName] = renderChart } - errList := new(multierror.Error) - for groupIndex, services := range productResp.Services { - var wg sync.WaitGroup - groupServices := make([]*commonmodels.ProductService, 0) - for _, service := range services { - if renderChart, isExist := renderChartMap[service.ServiceName]; isExist { - opt := &commonrepo.ServiceFindOption{ - ServiceName: service.ServiceName, - Type: service.Type, - Revision: service.Revision, - ProductName: productName, - //ExcludeStatus: setting.ProductStatusDeleting, - } - serviceObj, err := commonrepo.NewServiceColl().Find(opt) + prodServiceMap := productResp.GetServiceMap() + handler := func(serviceObj *commonmodels.Service, isRetry bool, log *zap.SugaredLogger) (err error) { + defer func() { + if prodSvc, ok := prodServiceMap[serviceObj.ServiceName]; ok { if err != nil { - log.Errorf("finded to find service %s, err %s", service.ServiceName, err.Error()) - continue + prodSvc.Error = err.Error() + } else { + prodSvc.Error = "" } - wg.Add(1) - go func(tmpRenderChart *template.RenderChart, currentService *commonmodels.Service) { - defer wg.Done() - - err = installOrUpgradeHelmChart(productResp.Namespace, renderChart, currentService, Timeout*time.Second*10, helmClient) - if err != nil { - errList = multierror.Append(errList, err) - return - } - }(renderChart, serviceObj) } - groupServices = append(groupServices, service) + }() + param, errBuildParam := buildInstallParam(productResp.Namespace, renderset.EnvName, renderset.DefaultValues, renderChartMap[serviceObj.ServiceName], serviceObj) + if errBuildParam != nil { + err = fmt.Errorf("failed to generate install param, service: %s, namespace: %s, err: %s", serviceObj.ServiceName, productResp.Namespace, errBuildParam) + return } - wg.Wait() - err = commonrepo.NewProductColl().UpdateGroup(envName, productName, groupIndex, groupServices) - if err != nil { - log.Errorf("Failed to update collection - service group %d. Error: %v", groupIndex, err) - return e.ErrUpdateEnv.AddDesc(err.Error()) + errInstall := installOrUpgradeHelmChartWithValues(param, isRetry, helmClient) + if errInstall != nil { + log.Errorf("failed to upgrade service: %s, namespace: %s, isRetry: %v, err: %s", serviceObj.ServiceName, productResp.Namespace, isRetry, errInstall) + err = fmt.Errorf("failed to upgrade service %s, err: %s", serviceObj.ServiceName, errInstall) } + return } - if err = commonrepo.NewProductColl().Update(productResp); err != nil { - errList = multierror.Append(errList, err) + + errList := new(multierror.Error) + for groupIndex, groupServices := range productResp.Services { + serviceList := make([]*commonmodels.Service, 0) + for _, service := range groupServices { + if _, ok := renderChartMap[service.ServiceName]; !ok { + continue + } + if filter != nil && !filter(service) { + continue + } + opt := &commonrepo.ServiceFindOption{ + ServiceName: service.ServiceName, + Type: service.Type, + Revision: service.Revision, + ProductName: productName, + } + serviceObj, err := commonrepo.NewServiceColl().Find(opt) + if err != nil { + log.Errorf("failed to find service %s, err %s", service.ServiceName, err.Error()) + return err + } + serviceList = append(serviceList, serviceObj) + } + groupServiceErr := batchExecutorWithRetry(3, time.Millisecond*500, serviceList, handler, log) + if groupServiceErr != nil { + errList = multierror.Append(errList, groupServiceErr...) + } + err := commonrepo.NewProductColl().UpdateGroup(envName, productName, groupIndex, groupServices) + if err != nil { + log.Errorf("Failed to update service group %d. Error: %v", groupIndex, err) + return err + } } return errList.ErrorOrNil() } + +func setFieldValueIsNotExist(obj map[string]interface{}, value interface{}, fields ...string) map[string]interface{} { + m := obj + for _, field := range fields[:len(fields)-1] { + if val, ok := m[field]; ok { + if valMap, ok := val.(map[string]interface{}); ok { + m = valMap + } else { + newVal := make(map[string]interface{}) + m[field] = newVal + m = newVal + } + } + } + m[fields[len(fields)-1]] = value + return obj +} + +func deploymentSelectorLabelExists(resourceName, namespace string, informer informers.SharedInformerFactory, log *zap.SugaredLogger) bool { + deployment, err := informer.Apps().V1().Deployments().Lister().Deployments(namespace).Get(resourceName) + // default we assume the deployment is new so we don't need to add selector labels + if err != nil { + if !apierrors.IsNotFound(err) { + log.Errorf("Failed to find deployment in the namespace: %s, the error is: %s", namespace, err) + } + return false + } + // since the 2 predefined labels are always together, we just check for only one + // if the match label exists, we return true. otherwise we return false + if _, ok := deployment.Spec.Selector.MatchLabels["s-product"]; ok { + return true + } + return false +} + +func statefulsetSelectorLabelExists(resourceName, namespace string, informer informers.SharedInformerFactory, log *zap.SugaredLogger) bool { + sts, err := informer.Apps().V1().StatefulSets().Lister().StatefulSets(namespace).Get(resourceName) + // default we assume the deployment is new so we don't need to add selector labels + if err != nil { + if !apierrors.IsNotFound(err) { + log.Errorf("Failed to find deployment in the namespace: %s, the error is: %s", namespace, err) + } + return false + } + // since the 2 predefined labels are always together, we just check for only one + // if the match label exists, we return true. otherwise we return false + if _, ok := sts.Spec.Selector.MatchLabels["s-product"]; ok { + return true + } + return false +} diff --git a/pkg/microservice/aslan/core/environment/service/environment_creator.go b/pkg/microservice/aslan/core/environment/service/environment_creator.go index 45a58c968995efa77061e7500dce1105593a37fe..c523a90c121bc39c99cd802298d91b4c79172e8e 100644 --- a/pkg/microservice/aslan/core/environment/service/environment_creator.go +++ b/pkg/microservice/aslan/core/environment/service/environment_creator.go @@ -19,20 +19,32 @@ package service import ( "errors" "fmt" + "strings" "time" + "github.com/hashicorp/go-multierror" "go.uber.org/zap" + versionedclient "istio.io/client-go/pkg/clientset/versioned" + "k8s.io/client-go/informers" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" "github.com/koderover/zadig/pkg/microservice/aslan/config" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" e "github.com/koderover/zadig/pkg/tool/errors" - "github.com/koderover/zadig/pkg/tool/helmclient" - "github.com/koderover/zadig/pkg/tool/kube/getter" + helmtool "github.com/koderover/zadig/pkg/tool/helmclient" + "github.com/koderover/zadig/pkg/tool/kube/informer" + "github.com/koderover/zadig/pkg/tool/kube/serializer" + "github.com/koderover/zadig/pkg/tool/kube/updater" + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types" ) type CreateProductParam struct { @@ -41,6 +53,7 @@ type CreateProductParam struct { ProductName string EnvType string log *zap.SugaredLogger + RegistryID string } type AutoCreator struct { @@ -74,7 +87,7 @@ func (autoCreator *AutoCreator) Create(envName string) (string, error) { return productResp.Status, nil } - productObject, err := GetInitProduct(productName, log) + productObject, err := GetInitProduct(productName, types.GeneralEnv, false, "", log) if err != nil { log.Errorf("AutoCreateProduct err:%v", err) return "", err @@ -84,6 +97,7 @@ func (autoCreator *AutoCreator) Create(envName string) (string, error) { productObject.Namespace = commonservice.GetProductEnvNamespace(envName, productName, "") productObject.UpdateBy = autoCreator.Param.UserName productObject.EnvName = envName + productObject.RegistryID = autoCreator.Param.RegistryID if autoCreator.Param.EnvType == setting.HelmDeployType { productObject.Source = setting.SourceFromHelm } @@ -120,35 +134,43 @@ func newHelmProductCreator() *HelmProductCreator { } func (creator *HelmProductCreator) Create(user, requestID string, args *models.Product, log *zap.SugaredLogger) error { - kubeClient, err := kube.GetKubeClient(args.ClusterID) + clusterID := args.ClusterID + if clusterID == "" { + projectClusterRelations, err := commonrepo.NewProjectClusterRelationColl().List(&commonrepo.ProjectClusterRelationOption{ + ProjectName: args.ProductName, + }) + if err != nil { + return e.ErrCreateEnv.AddDesc("Failed to get the cluster info selected in the project") + } + if len(projectClusterRelations) > 0 { + clusterID = projectClusterRelations[0].ClusterID + } + } + + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), clusterID) if err != nil { log.Errorf("[%s][%s] GetKubeClient error: %v", args.EnvName, args.ProductName, err) return e.ErrCreateEnv.AddErr(err) } + restConfig, err := kubeclient.GetRESTConfig(config.HubServerAddress(), clusterID) + if err != nil { + return e.ErrCreateEnv.AddErr(err) + } + + istioClient, err := versionedclient.NewForConfig(restConfig) + if err != nil { + return e.ErrCreateEnv.AddErr(err) + } + //判断namespace是否存在 namespace := args.GetNamespace() if args.Namespace == "" { args.Namespace = namespace } - _, found, err := getter.GetNamespace(args.Namespace, kubeClient) - if err != nil { - log.Errorf("GetNamespace error: %v", err) - return e.ErrCreateEnv.AddDesc(err.Error()) - } - if found { - log.Warnf("%s[%s]%s", "namespace", args.Namespace, "已经存在,请换个环境名称尝试!") - return e.ErrCreateEnv.AddDesc(fmt.Sprintf("%s[%s]%s", "namespace", args.Namespace, "已经存在,请换个环境名称尝试!")) - } - restConfig, err := kube.GetRESTConfig(args.ClusterID) - if err != nil { - log.Errorf("GetRESTConfig error: %v", err) - return e.ErrCreateEnv.AddDesc(err.Error()) - } - helmClient, err := helmclient.NewClientFromRestConf(restConfig, args.Namespace) + helmClient, err := helmtool.NewClientFromNamespace(args.ClusterID, args.Namespace) if err != nil { - log.Errorf("[%s][%s] NewClientFromRestConf error: %v", args.EnvName, args.ProductName, err) return e.ErrCreateEnv.AddErr(err) } @@ -156,7 +178,9 @@ func (creator *HelmProductCreator) Create(user, requestID string, args *models.P var renderSet *models.RenderSet if args.Render == nil || args.Render.Revision == 0 { renderSet, _, err = commonrepo.NewRenderSetColl().FindRenderSet(&commonrepo.RenderSetFindOption{ - Name: args.Namespace, + EnvName: args.EnvName, + Name: args.Namespace, + ProductTmpl: args.ProductName, }) if err != nil { @@ -191,21 +215,31 @@ func (creator *HelmProductCreator) Create(user, requestID string, args *models.P return e.ErrCreateEnv.AddDesc(err.Error()) } - if renderSet == nil { - renderSet, err = FindHelmRenderSet(args.ProductName, args.Render.Name, log) - if err != nil { - log.Errorf("[%s][P:%s] find product renderset error: %v", args.EnvName, args.ProductName, err) - return e.ErrCreateEnv.AddDesc(err.Error()) - } + renderSet, err = FindHelmRenderSet(args.ProductName, args.Render.Name, args.EnvName, log) + if err != nil { + log.Errorf("[%s][P:%s] find product renderset error: %v", args.EnvName, args.ProductName, err) + return e.ErrCreateEnv.AddDesc(err.Error()) + } + + // before create product, do install -dryRun to expose errors earlier + err = dryRunInstallRelease(args, renderSet, helmClient, log) + if err != nil { + log.Errorf("error occurred when installing services in env: %s/%s, err: %s ", args.ProductName, args.EnvName, err) + return e.ErrCreateEnv.AddErr(err) + } + + dryRunClient := client.NewDryRunClient(kubeClient) + err = initEnvConfigSetAction(args.EnvName, args.Namespace, args.ProductName, user, args.EnvConfigs, true, dryRunClient) + if err != nil { + log.Errorf("failed to dyrRun env resource [%s][P:%s], the error is: %s", args.EnvName, args.ProductName, err) + return e.ErrCreateEnv.AddErr(err) } - // 设置产品render revision args.Render.Revision = renderSet.Revision - // 记录服务当前对应render版本 setServiceRender(args) - args.Status = setting.ProductStatusCreating args.RecycleDay = config.DefaultRecycleDay() + args.ClusterID = clusterID if args.IsForkedProduct { args.RecycleDay = 7 } @@ -215,9 +249,15 @@ func (creator *HelmProductCreator) Create(user, requestID string, args *models.P return e.ErrCreateEnv.AddDesc(err.Error()) } - eventStart := time.Now().Unix() + err = initEnvConfigSetAction(args.EnvName, args.Namespace, args.ProductName, user, args.EnvConfigs, false, kubeClient) + if err != nil { + log.Errorf("failed to helmInitEnvConfigSet [%s][P:%s], the error is: %s", args.EnvName, args.ProductName, err) + if err := commonrepo.NewProductColl().UpdateStatusAndError(args.EnvName, args.ProductName, setting.ProductStatusFailed, err.Error()); err != nil { + log.Errorf("helmInitEnvConfigSet [%s][P:%s] Product.UpdateStatus error: %s", args.EnvName, args.ProductName, err) + } + } - go installProductHelmCharts(user, args.EnvName, requestID, args, eventStart, helmClient, log) + go installProductHelmCharts(user, args.EnvName, requestID, args, renderSet, time.Now().Unix(), helmClient, kubeClient, istioClient, log) return nil } @@ -255,8 +295,6 @@ func (creator *PMProductCreator) Create(user, requestID string, args *models.Pro return e.ErrCreateEnv.AddDesc(err.Error()) } - eventStart := time.Now().Unix() - args.Status = setting.ProductStatusCreating args.RecycleDay = config.DefaultRecycleDay() err := commonrepo.NewProductColl().Create(args) @@ -265,7 +303,7 @@ func (creator *PMProductCreator) Create(user, requestID string, args *models.Pro return e.ErrCreateEnv.AddDesc(err.Error()) } // 异步创建产品 - go createGroups(args.EnvName, user, requestID, args, eventStart, nil, nil, log) + go createGroups(args.EnvName, user, requestID, args, time.Now().Unix(), nil, nil, nil, nil, log) return nil } @@ -276,25 +314,61 @@ func newDefaultProductCreator() *DefaultProductCreator { return &DefaultProductCreator{} } +func dryRunServices(args *commonmodels.Product, renderSet *commonmodels.RenderSet, informer informers.SharedInformerFactory, kubeClient client.Client, log *zap.SugaredLogger) error { + errList := &multierror.Error{} + for _, group := range args.Services { + for _, svc := range group { + _, err := upsertService(false, args, svc, nil, renderSet, informer, kubeClient, nil, log) + if err != nil { + errList = multierror.Append(errList, fmt.Errorf("failed to dryRun apply service: %s, err: %s", svc.ServiceName, err)) + } + } + } + return errList.ErrorOrNil() +} + func (creator *DefaultProductCreator) Create(user, requestID string, args *models.Product, log *zap.SugaredLogger) error { - kubeClient, err := kube.GetKubeClient(args.ClusterID) + // get project cluster relation + clusterID := args.ClusterID + if clusterID == "" { + projectClusterRelations, err := commonrepo.NewProjectClusterRelationColl().List(&commonrepo.ProjectClusterRelationOption{ + ProjectName: args.ProductName, + }) + if err != nil { + return e.ErrCreateEnv.AddDesc("Failed to get the cluster info selected in the project") + } + if len(projectClusterRelations) > 0 { + clusterID = projectClusterRelations[0].ClusterID + } + } + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), clusterID) if err != nil { return e.ErrCreateEnv.AddErr(err) } + cls, err := kubeclient.GetKubeClientSet(config.HubServerAddress(), clusterID) + if err != nil { + return e.ErrCreateEnv.AddErr(err) + } + inf, err := informer.NewInformer(clusterID, args.Namespace, cls) + if err != nil { + return e.ErrCreateEnv.AddErr(err) + } + + restConfig, err := kubeclient.GetRESTConfig(config.HubServerAddress(), clusterID) + if err != nil { + return fmt.Errorf("failed to get rest config: %s", err) + } + istioClient, err := versionedclient.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("failed to new istio client: %s", err) + } + //判断namespace是否存在 namespace := args.GetNamespace() if args.Namespace == "" { args.Namespace = namespace } - _, found, err := getter.GetNamespace(args.Namespace, kubeClient) - if err != nil { - log.Errorf("GetNamespace error: %v", err) - return e.ErrCreateEnv.AddDesc(err.Error()) - } - if found { - return e.ErrCreateEnv.AddDesc(fmt.Sprintf("%s[%s]%s", "namespace", args.Namespace, "已经存在,请换个环境名称尝试!")) - } //创建角色环境之间的关联关系 //todo 创建环境暂时不指定角色 @@ -304,8 +378,6 @@ func (creator *DefaultProductCreator) Create(user, requestID string, args *model return e.ErrCreateEnv.AddDesc(err.Error()) } - eventStart := time.Now().Unix() - // 检查renderinfo是否为空，避免空指针 if args.Render == nil { args.Render = &models.RenderInfo{ProductTmpl: args.ProductName} } @@ -318,26 +390,122 @@ func (creator *DefaultProductCreator) Create(user, requestID string, args *model // 如果是版本回滚，则args.Render.Revision > 0 if args.Render.Revision == 0 { // 检查renderset是否覆盖产品所有key - renderSet, err = commonservice.ValidateRenderSet(args.ProductName, args.Render.Name, nil, log) + renderSet, err = commonservice.ValidateRenderSet(args.ProductName, args.Render.Name, args.EnvName, nil, log) if err != nil { log.Errorf("[%s][P:%s] validate product renderset error: %v", args.EnvName, args.ProductName, err) return e.ErrCreateEnv.AddDesc(err.Error()) } - // 保存产品信息,并设置产品状态 - // 设置产品render revsion args.Render.Revision = renderSet.Revision - // 记录服务当前对应render版本 setServiceRender(args) } + // before we apply yaml to k8s, we run kubectl apply --dry-run to expose problems early + dryRunClient := client.NewDryRunClient(kubeClient) + err = dryRunServices(args, renderSet, inf, dryRunClient, log) + if err != nil { + return e.ErrCreateEnv.AddErr(err) + } + + err = initEnvConfigSetAction(args.EnvName, args.Namespace, args.ProductName, user, args.EnvConfigs, true, dryRunClient) + if err != nil { + return e.ErrCreateEnv.AddErr(err) + } + args.Status = setting.ProductStatusCreating args.RecycleDay = config.DefaultRecycleDay() + args.ClusterID = clusterID err = commonrepo.NewProductColl().Create(args) if err != nil { log.Errorf("[%s][%s] create product record error: %v", args.EnvName, args.ProductName, err) return e.ErrCreateEnv.AddDesc(err.Error()) } - // 异步创建产品 - go createGroups(args.EnvName, user, requestID, args, eventStart, renderSet, kubeClient, log) + + go createGroups(args.EnvName, user, requestID, args, time.Now().Unix(), renderSet, inf, kubeClient, istioClient, log) return nil } + +func initEnvConfigSetAction(envName, namespace, productName, userName string, envResources []*models.CreateUpdateCommonEnvCfgArgs, dryRun bool, kubeClient client.Client) error { + errList := &multierror.Error{ + ErrorFormat: func(es []error) string { + format := "创建环境配置" + if len(es) == 1 { + return fmt.Sprintf(format+" %s 失败:%s", envName, es[0]) + } + points := make([]string, len(es)) + for i, err := range es { + points[i] = fmt.Sprintf("* %s", err) + } + return fmt.Sprintf(format+" %s 失败:\n%s", envName, strings.Join(points, "\n")) + }, + } + + clusterLabels := getPredefinedClusterLabels(productName, "", envName) + delete(clusterLabels, "s-service") + + for _, envResource := range envResources { + u, err := serializer.NewDecoder().YamlToUnstructured([]byte(envResource.YamlData)) + if err != nil { + log.Errorf("Failed to convert yaml to Unstructured, manifest is\n%s\n, error: %s", envResource.YamlData, err) + errList = multierror.Append(errList, err) + continue + } + switch u.GetKind() { + case setting.ConfigMap, setting.Ingress, setting.Secret, setting.PersistentVolumeClaim: + ls := kube.MergeLabels(clusterLabels, u.GetLabels()) + u.SetNamespace(namespace) + u.SetLabels(ls) + _, err := ensureLabelAndNs(u, namespace, productName) + if err != nil { + errList = multierror.Append(errList, err) + continue + } + + err = updater.CreateOrPatchUnstructuredNeverAnnotation(u, kubeClient) + if err != nil { + log.Errorf("Failed to initEnvConfigSet %s, manifest is\n%v\n, error: %s", u.GetKind(), u, err) + errList = multierror.Append(errList, err) + continue + } + + if dryRun { + continue + } + + u.SetManagedFields(nil) + yamlData, err := yaml.Marshal(u.UnstructuredContent()) + if err != nil { + log.Errorf("Failed to initEnvConfigSet yaml.Marshal %s, manifest is\n%v\n, error: %s", u.GetKind(), u, err) + errList = multierror.Append(errList, err) + continue + } + envResourceObj := &models.EnvResource{ + ProductName: productName, + UpdateUserName: userName, + EnvName: envName, + Namespace: namespace, + Name: u.GetName(), + YamlData: string(yamlData), + SourceDetail: geneSourceDetail(envResource.GitRepoConfig), + AutoSync: envResource.AutoSync, + } + + switch u.GetKind() { + case setting.ConfigMap: + envResourceObj.Type = string(config.CommonEnvCfgTypeConfigMap) + case setting.Ingress: + envResourceObj.Type = string(config.CommonEnvCfgTypeIngress) + case setting.Secret: + envResourceObj.Type = string(config.CommonEnvCfgTypeSecret) + case setting.PersistentVolumeClaim: + envResourceObj.Type = string(config.CommonEnvCfgTypePvc) + } + if err := commonrepo.NewEnvResourceColl().Create(envResourceObj); err != nil { + errList = multierror.Append(errList, err) + } + default: + errList = multierror.Append(errList, fmt.Errorf("Failed to initEnvConfigSet %s, manifest is\n%v\n, error: %s", u.GetKind(), u, "kind not support")) + } + } + + return errList.ErrorOrNil() +} diff --git a/pkg/microservice/aslan/core/environment/service/environment_group.go b/pkg/microservice/aslan/core/environment/service/environment_group.go index 70e8f6890a044400f6a0897c1fb6d1539c3eac41..5809d02d083a1d388fd95cf9cc09e1f110969a43 100644 --- a/pkg/microservice/aslan/core/environment/service/environment_group.go +++ b/pkg/microservice/aslan/core/environment/service/environment_group.go @@ -22,16 +22,20 @@ import ( "go.uber.org/zap" "helm.sh/helm/v3/pkg/releaseutil" - "sigs.k8s.io/controller-runtime/pkg/client" + "k8s.io/client-go/informers" + "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" "github.com/koderover/zadig/pkg/shared/kube/resource" "github.com/koderover/zadig/pkg/shared/kube/wrapper" e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/kube/getter" + "github.com/koderover/zadig/pkg/tool/kube/informer" "github.com/koderover/zadig/pkg/tool/kube/serializer" "github.com/koderover/zadig/pkg/util" ) @@ -65,7 +69,12 @@ func ListGroups(serviceName, envName, productName string, perPage, page int, log //将获取到的所有服务按照名称进行排序 sort.SliceStable(allServices, func(i, j int) bool { return allServices[i].ServiceName < allServices[j].ServiceName }) - kubeClient, err := kube.GetKubeClient(productInfo.ClusterID) + cls, err := kubeclient.GetKubeClientSet(config.HubServerAddress(), productInfo.ClusterID) + if err != nil { + log.Errorf("[%s][%s] error: %v", envName, productName, err) + return resp, count, e.ErrListGroups.AddDesc(err.Error()) + } + inf, err := informer.NewInformer(productInfo.ClusterID, productInfo.Namespace, cls) if err != nil { log.Errorf("[%s][%s] error: %v", envName, productName, err) return resp, count, e.ErrListGroups.AddDesc(err.Error()) @@ -73,14 +82,14 @@ func ListGroups(serviceName, envName, productName string, perPage, page int, log //这种针对的是获取所有数据的接口，内部调用 if page == 0 && perPage == 0 { - resp = envHandleFunc(getProjectType(productName), log).listGroupServices(allServices, envName, productName, kubeClient, productInfo) + resp = envHandleFunc(getProjectType(productName), log).listGroupServices(allServices, envName, productName, inf, productInfo) return resp, count, nil } //针对获取环境状态的接口请求，这里不需要一次性获取所有的服务，先获取十条的数据，有异常的可以直接返回，不需要继续往下获取 if page == -1 && perPage == -1 { // 获取环境的状态 - resp = listGroupServiceStatus(allServices, envName, productName, kubeClient, productInfo, log) + resp = listGroupServiceStatus(allServices, envName, productName, inf, productInfo, log) return resp, count, nil } @@ -93,11 +102,11 @@ func ListGroups(serviceName, envName, productName string, perPage, page int, log } currentServices = allServices[currentPage*perPage:] } - resp = envHandleFunc(getProjectType(productName), log).listGroupServices(currentServices, envName, productName, kubeClient, productInfo) + resp = envHandleFunc(getProjectType(productName), log).listGroupServices(currentServices, envName, productName, inf, productInfo) return resp, count, nil } -func listGroupServiceStatus(allServices []*commonmodels.ProductService, envName, productName string, kubeClient client.Client, productInfo *commonmodels.Product, log *zap.SugaredLogger) []*commonservice.ServiceResp { +func listGroupServiceStatus(allServices []*commonmodels.ProductService, envName, productName string, informer informers.SharedInformerFactory, productInfo *commonmodels.Product, log *zap.SugaredLogger) []*commonservice.ServiceResp { var ( count = len(allServices) currentServices = make([]*commonmodels.ProductService, 0) @@ -110,7 +119,7 @@ func listGroupServiceStatus(allServices []*commonmodels.ProductService, envName, } else { currentServices = allServices[page*perPage:] } - resp = envHandleFunc(getProjectType(productName), log).listGroupServices(currentServices, envName, productName, kubeClient, productInfo) + resp = envHandleFunc(getProjectType(productName), log).listGroupServices(currentServices, envName, productName, informer, productInfo) allRunning := true for _, serviceResp := range resp { // Service是物理机部署时，无需判断状态 @@ -136,7 +145,7 @@ func GetIngressInfo(product *commonmodels.Product, service *commonmodels.Service renderSet := &commonmodels.RenderSet{} if product.Render != nil { - renderSet, err = commonservice.GetRenderSet(product.Render.Name, 0, log) + renderSet, err = commonservice.GetRenderSet(product.Render.Name, 0, false, product.EnvName, log) if err != nil { log.Errorf("GetRenderSet %s error: %v", product.ProductName, err) return ingressInfo @@ -149,13 +158,52 @@ func GetIngressInfo(product *commonmodels.Product, service *commonmodels.Service parsedYaml = util.ReplaceWrapLine(parsedYaml) yamlContentArray := releaseutil.SplitManifests(parsedYaml) hostInfos := make([]resource.HostInfo, 0) + for _, item := range yamlContentArray { - ing, err := serializer.NewDecoder().YamlToIngress([]byte(item)) - if err != nil || ing == nil { + u, err := serializer.NewDecoder().YamlToUnstructured([]byte(item)) + if err != nil { + log.Warnf("Failed to decode yaml to Unstructured, err: %s", err) continue } + switch u.GetKind() { + case setting.Ingress: + clientset, err := kubeclient.GetKubeClientSet(config.HubServerAddress(), product.ClusterID) + if err != nil { + log.Errorf("failed to init clientset, clusterID: %s", product.ClusterID) + return nil + } - hostInfos = append(hostInfos, wrapper.Ingress(ing).HostInfo()...) + inf, err := informer.NewInformer(product.ClusterID, product.Namespace, clientset) + if err != nil { + log.Errorf("failed to create informer from clientset for clusterID: %s, the error is: %s", product.ClusterID, err) + return nil + } + + version, err := clientset.Discovery().ServerVersion() + if err != nil { + log.Warnf("Failed to determine server version, error is: %s", err) + continue + } + + if kubeclient.VersionLessThan122(version) { + // get the ingress info from kubernetes. For cluster version 1.22- we only search for + // extensions/v1beta1. + // FIXME: add networking.k8s.io/v1beta1 & networking.k8s.io/v1 support for cluster 1.22- + ing, found, err := getter.GetExtensionsV1Beta1Ingress(product.Namespace, u.GetName(), inf) + if err != nil || !found { + log.Warnf("no ingress %s found in %s:%s %v", u.GetName(), service.ServiceName, product.Namespace, err) + continue + } + hostInfos = append(hostInfos, wrapper.Ingress(ing).HostInfo()...) + } else { + ing, err := getter.GetNetworkingV1Ingress(product.Namespace, u.GetName(), inf) + if err != nil { + log.Warnf("no ingress %s found in %s:%s %v", u.GetName(), service.ServiceName, product.Namespace, err) + continue + } + hostInfos = append(hostInfos, wrapper.GetIngressHostInfo(ing)...) + } + } } ingressInfo.HostInfo = hostInfos return ingressInfo diff --git a/pkg/microservice/aslan/core/environment/service/environment_update.go b/pkg/microservice/aslan/core/environment/service/environment_update.go new file mode 100644 index 0000000000000000000000000000000000000000..b0a0bff43eb49ad0cf11c6c76a90c7b248316038 --- /dev/null +++ b/pkg/microservice/aslan/core/environment/service/environment_update.go @@ -0,0 +1,367 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "context" + "fmt" + "time" + + "github.com/hashicorp/go-multierror" + + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + templatemodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" + commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + helmtool "github.com/koderover/zadig/pkg/tool/helmclient" + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/util" + helmclient "github.com/mittwald/go-helm-client" + "github.com/pkg/errors" + "go.uber.org/zap" +) + +func UninstallServiceByName(helmClient helmclient.Client, serviceName string, env *commonmodels.Product, revision int64, force bool) error { + revisionSvc, err := commonrepo.NewServiceColl().Find(&commonrepo.ServiceFindOption{ + ServiceName: serviceName, + Revision: revision, + ProductName: env.ProductName, + }) + if err != nil { + return fmt.Errorf("failed to find service: %s with revision: %d, err: %s", serviceName, revision, err) + } + return UninstallService(helmClient, env, revisionSvc, force) +} + +func UninstallService(helmClient helmclient.Client, env *commonmodels.Product, revisionSvc *commonmodels.Service, force bool) error { + releaseName := util.GeneReleaseName(revisionSvc.GetReleaseNaming(), env.ProductName, env.Namespace, env.EnvName, revisionSvc.ServiceName) + + err := EnsureDeleteZadigServiceByHelmRelease(context.TODO(), env, releaseName, helmClient) + if err != nil { + return fmt.Errorf("failed to ensure delete Zadig Service by helm release: %s", err) + } + + return helmClient.UninstallRelease(&helmclient.ChartSpec{ + ReleaseName: releaseName, + Namespace: env.Namespace, + Wait: true, + Force: force, + Timeout: time.Second * 10, + }) +} + +func InstallService(helmClient *helmtool.HelmClient, param *ReleaseInstallParam) error { + handler := func(serviceObj *commonmodels.Service, isRetry bool, log *zap.SugaredLogger) error { + errInstall := installOrUpgradeHelmChartWithValues(param, isRetry, helmClient) + if errInstall != nil { + log.Errorf("failed to upgrade service: %s, namespace: %s, isRetry: %v, err: %s", serviceObj.ServiceName, param.Namespace, isRetry, errInstall) + return errors.Wrapf(errInstall, "failed to install or upgrade service %s", serviceObj.ServiceName) + } + return nil + } + errList := batchExecutorWithRetry(3, time.Millisecond*2500, []*commonmodels.Service{param.serviceObj}, handler, log.SugaredLogger()) + if len(errList) > 0 { + return errList[0] + } + return nil +} + +func setProductServiceError(productInfo *commonmodels.Product, serviceName string, err error) error { + foundSvc := false + // update service revision data in product + for groupIndex, svcGroup := range productInfo.Services { + for _, svc := range svcGroup { + if svc.ServiceName == serviceName { + if err != nil { + svc.Error = err.Error() + } else { + svc.Error = "" + } + foundSvc = true + break + } + } + if foundSvc { + if err := commonrepo.NewProductColl().UpdateGroup(productInfo.EnvName, productInfo.ProductName, groupIndex, svcGroup); err != nil { + return fmt.Errorf("failed to update service group: %s:%s, err: %s ", productInfo.ProductName, productInfo.EnvName, err) + } + break + } + } + return nil +} + +func updateServiceRevisionInProduct(productInfo *commonmodels.Product, serviceName string, serviceRevision int64) error { + foundSvc := false + // update service revision data in product + for groupIndex, svcGroup := range productInfo.Services { + for _, svc := range svcGroup { + if svc.ServiceName == serviceName { + svc.Revision = serviceRevision + foundSvc = true + break + } + } + if foundSvc { + if err := commonrepo.NewProductColl().UpdateGroup(productInfo.EnvName, productInfo.ProductName, groupIndex, svcGroup); err != nil { + return fmt.Errorf("failed to update service group: %s:%s, err: %s ", productInfo.ProductName, productInfo.EnvName, err) + } + break + } + } + return nil +} + +// reInstallHelmServiceInEnv uninstall the service and reinstall to update releaseNaming rule +func reInstallHelmServiceInEnv(productInfo *commonmodels.Product, templateSvc *commonmodels.Service) (finalError error) { + productSvcMap, serviceName := productInfo.GetServiceMap(), templateSvc.ServiceName + productSvc, ok := productSvcMap[serviceName] + // service not applied in this environment + if !ok { + return nil + } + + var err error + defer func() { + finalError = setProductServiceError(productInfo, templateSvc.ServiceName, err) + }() + + renderInfo, errRender := commonrepo.NewRenderSetColl().Find(&commonrepo.RenderSetFindOption{ + ProductTmpl: productInfo.ProductName, + EnvName: productInfo.EnvName, + Name: productInfo.Render.Name, + Revision: productInfo.Render.Revision}) + if errRender != nil { + err = fmt.Errorf("failed to find renderset, %s:%d, err: %s", productInfo.Render.Name, productInfo.Render.Revision, errRender) + return + } + + var renderChart *templatemodels.RenderChart + for _, _renderChart := range renderInfo.ChartInfos { + if _renderChart.ServiceName == serviceName { + renderChart = _renderChart + } + } + if renderChart == nil { + err = fmt.Errorf("failed to find renderchart: %s", renderChart.ServiceName) + return + } + + helmClient, errCreateClient := helmtool.NewClientFromNamespace(productInfo.ClusterID, productInfo.Namespace) + if errCreateClient != nil { + err = fmt.Errorf("failed to create helm client, err: %s", errCreateClient) + return + } + + prodSvcTemp, errFindProd := commonrepo.NewServiceColl().Find(&commonrepo.ServiceFindOption{ServiceName: serviceName, Revision: productSvc.Revision, ProductName: productInfo.ProductName}) + if errFindProd != nil { + err = fmt.Errorf("failed to get service: %s with revision: %d, err: %s", serviceName, productSvc.Revision, errFindProd) + return + } + + // nothing would happen if release naming rule are same + if prodSvcTemp.GetReleaseNaming() == templateSvc.GetReleaseNaming() { + return nil + } + + if err = UninstallService(helmClient, productInfo, prodSvcTemp, true); err != nil { + err = fmt.Errorf("helm uninstall service: %s in namespace: %s, err: %s", serviceName, productInfo.Namespace, err) + return + } + + param, errBuildParam := buildInstallParam(productInfo.Namespace, productInfo.EnvName, renderInfo.DefaultValues, renderChart, templateSvc) + if errBuildParam != nil { + err = fmt.Errorf("failed to generate install param, service: %s, namespace: %s, err: %s", templateSvc.ServiceName, productInfo.Namespace, errBuildParam) + return + } + + err = InstallService(helmClient, param) + if err != nil { + return + } + + err = updateServiceRevisionInProduct(productInfo, templateSvc.ServiceName, templateSvc.Revision) + return +} + +func updateHelmServiceInEnv(userName string, productInfo *commonmodels.Product, templateSvc *commonmodels.Service) (finalError error) { + productSvcMap, serviceName := productInfo.GetServiceMap(), templateSvc.ServiceName + // service not applied in this environment + if _, ok := productSvcMap[serviceName]; !ok { + return nil + } + + var err error + defer func() { + finalError = setProductServiceError(productInfo, templateSvc.ServiceName, err) + }() + + renderInfo, errFindRender := commonrepo.NewRenderSetColl().Find(&commonrepo.RenderSetFindOption{ + ProductTmpl: productInfo.ProductName, + EnvName: productInfo.EnvName, + Name: productInfo.Render.Name, + Revision: productInfo.Render.Revision}) + if errFindRender != nil { + err = fmt.Errorf("failed to find renderset, %s:%d, err: %s", productInfo.Render.Name, productInfo.Render.Revision, errFindRender) + return + } + + var renderChart *templatemodels.RenderChart + for _, _renderChart := range renderInfo.ChartInfos { + if _renderChart.ServiceName == serviceName { + renderChart = _renderChart + } + } + if renderChart == nil { + err = fmt.Errorf("failed to find renderchart for service: %s", renderChart.ServiceName) + return + } + + chartArg := &commonservice.RenderChartArg{} + chartArg.FillRenderChartModel(renderChart, renderChart.ChartVersion) + newRenderSet, errDiffRender := diffRenderSet(userName, productInfo.ProductName, productInfo.EnvName, productInfo, []*commonservice.RenderChartArg{chartArg}, log.SugaredLogger()) + if errDiffRender != nil { + err = fmt.Errorf("failed to build renderset for service: %s, err: %s", renderChart.ServiceName, errDiffRender) + return + } + + // update renderChart + for _, _renderChart := range newRenderSet.ChartInfos { + if _renderChart.ServiceName == serviceName { + renderChart = _renderChart + } + } + + helmClient, errCreateClient := helmtool.NewClientFromNamespace(productInfo.ClusterID, productInfo.Namespace) + if errCreateClient != nil { + err = fmt.Errorf("failed to create helm client, err: %s", errCreateClient) + return + } + + param, errBuildParam := buildInstallParam(productInfo.Namespace, productInfo.EnvName, renderInfo.DefaultValues, renderChart, templateSvc) + if err != nil { + return fmt.Errorf("failed to generate install param, service: %s err: %s", templateSvc.ServiceName, errBuildParam) + } + + err = InstallService(helmClient, param) + if err != nil { + return fmt.Errorf("fauled to install release for service: %s, err: %s", templateSvc.ServiceName, err) + } + + err = updateServiceRevisionInProduct(productInfo, templateSvc.ServiceName, templateSvc.Revision) + return +} + +func updateK8sServiceInEnv(productInfo *commonmodels.Product, templateSvc *commonmodels.Service) error { + productSvcMap, serviceName := productInfo.GetServiceMap(), templateSvc.ServiceName + // service not applied in this environment + if _, ok := productSvcMap[serviceName]; !ok { + return nil + } + return UpdateProductV2(productInfo.EnvName, productInfo.ProductName, "", "", []string{templateSvc.ServiceName}, false, nil, log.SugaredLogger()) +} + +// ReInstallHelmSvcInAllEnvs reinstall svc in all envs in which the svc is already installed +func ReInstallHelmSvcInAllEnvs(productName string, templateSvc *commonmodels.Service) error { + products, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{ + Name: productName, + }) + if err != nil { + return fmt.Errorf("failed to list envs for product: %s, err: %s", productName, err) + } + retErr := &multierror.Error{} + for _, product := range products { + err := reInstallHelmServiceInEnv(product, templateSvc) + if err != nil { + retErr = multierror.Append(retErr, fmt.Errorf("failed to update service: %s/%s, err: %s", product.EnvName, templateSvc.ServiceName, err)) + } + } + return retErr.ErrorOrNil() +} + +// updateHelmSvcInAllEnvs updates helm svc in all envs in which the svc is already installed +func updateHelmSvcInAllEnvs(userName, productName string, templateSvcs []*commonmodels.Service) error { + products, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{ + Name: productName, + }) + if err != nil { + return fmt.Errorf("failed to list envs for product: %s, err: %s", productName, err) + } + retErr := &multierror.Error{} + for _, product := range products { + for _, templateSvc := range templateSvcs { + err := updateHelmServiceInEnv(userName, product, templateSvc) + if err != nil { + retErr = multierror.Append(retErr, fmt.Errorf("failed to update service: %s/%s, err: %s", product.EnvName, templateSvc.ServiceName, err)) + } + } + } + return retErr.ErrorOrNil() +} + +// updateK8sSvcInAllEnvs updates k8s svc in all envs in which the svc is already installed +func updateK8sSvcInAllEnvs(productName string, templateSvc *commonmodels.Service) error { + products, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{ + Name: productName, + }) + if err != nil { + return fmt.Errorf("failed to list envs for product: %s, err: %s", productName, err) + } + retErr := &multierror.Error{} + for _, product := range products { + err := updateK8sServiceInEnv(product, templateSvc) + if err != nil { + retErr = multierror.Append(retErr, err) + } + } + return retErr.ErrorOrNil() +} + +func AutoDeployHelmServiceToEnvs(userName, requestID, projectName string, serviceTemplates []*commonmodels.Service, log *zap.SugaredLogger) error { + templateProduct, err := templaterepo.NewProductColl().Find(projectName) + if err != nil { + return fmt.Errorf("failed to find template product when depolying services: %s, err: %s", projectName, err) + } + if templateProduct.AutoDeploy == nil || !templateProduct.AutoDeploy.Enable { + return nil + } + go func() { + err = updateHelmSvcInAllEnvs(userName, projectName, serviceTemplates) + if err != nil { + commonservice.SendErrorMessage(userName, "服务自动部署失败", requestID, err, log) + } + }() + return nil +} + +func AutoDeployYamlServiceToEnvs(userName, requestID string, serviceTemplate *commonmodels.Service, log *zap.SugaredLogger) error { + templateProduct, err := templaterepo.NewProductColl().Find(serviceTemplate.ProductName) + if err != nil { + return fmt.Errorf("failed to find template product when depolying services: %s, err: %s", serviceTemplate.ServiceName, err) + } + if templateProduct.AutoDeploy == nil || !templateProduct.AutoDeploy.Enable { + return nil + } + go func() { + err = updateK8sSvcInAllEnvs(serviceTemplate.ProductName, serviceTemplate) + if err != nil { + commonservice.SendErrorMessage(userName, "服务自动部署失败", requestID, err, log) + } + }() + return nil +} diff --git a/pkg/microservice/aslan/core/environment/service/export.go b/pkg/microservice/aslan/core/environment/service/export.go index 035c30d5b1c65abd80e5839ff1132d2bf4ba2798..acf9aa6b0959e5f7ecf2bc8e7cd9b8a52e1fa513 100644 --- a/pkg/microservice/aslan/core/environment/service/export.go +++ b/pkg/microservice/aslan/core/environment/service/export.go @@ -21,9 +21,10 @@ import ( "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/koderover/zadig/pkg/microservice/aslan/config" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" "github.com/koderover/zadig/pkg/tool/kube/getter" ) @@ -39,7 +40,7 @@ func ExportYaml(envName, productName, serviceName string, log *zap.SugaredLogger } namespace := env.Namespace - kubeClient, err := kube.GetKubeClient(env.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), env.ClusterID) if err != nil { log.Errorf("cluster is not connected [%s][%s][%s]", env.EnvName, env.ProductName, env.ClusterID) return res diff --git a/pkg/microservice/aslan/core/environment/service/helm.go b/pkg/microservice/aslan/core/environment/service/helm.go new file mode 100644 index 0000000000000000000000000000000000000000..fdfe2a7b0956c7011dcac8d0418344e285b2a6cc --- /dev/null +++ b/pkg/microservice/aslan/core/environment/service/helm.go @@ -0,0 +1,612 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/hashicorp/go-multierror" + "github.com/otiai10/copy" + "github.com/pkg/errors" + "go.uber.org/zap" + "helm.sh/helm/v3/pkg/action" + "helm.sh/helm/v3/pkg/release" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/yaml" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" + "github.com/koderover/zadig/pkg/setting" + e "github.com/koderover/zadig/pkg/tool/errors" + helmtool "github.com/koderover/zadig/pkg/tool/helmclient" + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types" + "github.com/koderover/zadig/pkg/util" + "github.com/koderover/zadig/pkg/util/converter" + jsonutil "github.com/koderover/zadig/pkg/util/json" +) + +type HelmReleaseQueryArgs struct { + ProjectName string `json:"projectName" form:"projectName"` + Filter string `json:"filter" form:"filter"` +} + +type ReleaseFilter struct { + Name string `json:"name"` + Status ReleaseStatus `json:"status"` +} + +type HelmReleaseResp struct { + ReleaseName string `json:"releaseName"` + ServiceName string `json:"serviceName"` + Revision int `json:"revision"` + Chart string `json:"chart"` + AppVersion string `json:"appVersion"` + Status ReleaseStatus `json:"status"` + Updatable bool `json:"updatable"` + Error string `json:"error"` +} + +type ChartInfo struct { + ServiceName string `json:"serviceName"` + Revision int64 `json:"revision"` +} + +type HelmChartsResp struct { + ChartInfos []*ChartInfo `json:"chartInfos"` + FileInfos []*types.FileInfo `json:"fileInfos"` +} + +type ValuesResp struct { + ValuesYaml string `json:"valuesYaml"` +} + +type SvcDataSet struct { + ProdSvc *models.ProductService + TmplSvc *models.Service + SvcRelease *release.Release +} + +type ReleaseStatus string + +const ( + HelmReleaseStatusPending ReleaseStatus = "pending" + HelmReleaseStatusDeployed ReleaseStatus = "deployed" + HelmReleaseStatusFailed ReleaseStatus = "failed" + HelmReleaseStatusNotDeployed ReleaseStatus = "notDeployed" +) + +func listReleaseInNamespace(helmClient *helmtool.HelmClient, filter *ReleaseFilter) ([]*release.Release, error) { + listClient := action.NewList(helmClient.ActionConfig) + + switch filter.Status { + case HelmReleaseStatusDeployed: + listClient.StateMask = action.ListDeployed + case HelmReleaseStatusFailed: + listClient.StateMask = action.ListFailed | action.ListSuperseded + case HelmReleaseStatusPending: + listClient.StateMask = action.ListPendingRollback | action.ListPendingUpgrade | action.ListPendingInstall | action.ListUninstalling + default: + listClient.StateMask = action.ListAll + } + listClient.Filter = filter.Name + + return listClient.Run() +} + +func getReleaseStatus(re *release.Release) ReleaseStatus { + switch re.Info.Status { + case release.StatusDeployed: + return HelmReleaseStatusDeployed + case release.StatusFailed, release.StatusSuperseded: + return HelmReleaseStatusFailed + case release.StatusPendingRollback, release.StatusPendingUpgrade, release.StatusPendingInstall, release.StatusUninstalling: + return HelmReleaseStatusPending + default: + return HelmReleaseStatusFailed + } +} + +type ImageData struct { + ImageName string `json:"imageName"` + ImageTag string `json:"imageTag"` + Selected bool `json:"selected"` +} + +type ServiceImages struct { + ServiceName string `json:"serviceName"` + Images []*ImageData `json:"imageData"` +} + +type ChartImagesResp struct { + ServiceImages []*ServiceImages `json:"serviceImages"` +} + +func GetChartValues(projectName, envName, serviceName string) (*ValuesResp, error) { + opt := &commonrepo.ProductFindOptions{Name: projectName, EnvName: envName} + prod, err := commonrepo.NewProductColl().Find(opt) + if err != nil { + return nil, fmt.Errorf("failed to find project: %s, err: %s", projectName, err) + } + + restConfig, err := kube.GetRESTConfig(prod.ClusterID) + if err != nil { + log.Errorf("GetRESTConfig error: %s", err) + return nil, fmt.Errorf("failed to get k8s rest config, err: %s", err) + } + helmClient, err := helmtool.NewClientFromRestConf(restConfig, prod.Namespace) + if err != nil { + log.Errorf("[%s][%s] NewClientFromRestConf error: %s", envName, projectName, err) + return nil, fmt.Errorf("failed to init helm client, err: %s", err) + } + + serviceMap := prod.GetServiceMap() + prodSvc, ok := serviceMap[serviceName] + if !ok { + return nil, fmt.Errorf("failed to find sercice: %s in env: %s", serviceName, envName) + } + + revisionSvc, err := commonrepo.NewServiceColl().Find(&commonrepo.ServiceFindOption{ + ServiceName: serviceName, + Revision: prodSvc.Revision, + ProductName: prodSvc.ProductName, + }) + if err != nil { + return nil, err + } + + releaseName := util.GeneReleaseName(revisionSvc.GetReleaseNaming(), prodSvc.ProductName, prod.Namespace, prod.EnvName, prodSvc.ServiceName) + valuesMap, err := helmClient.GetReleaseValues(releaseName, true) + if err != nil { + log.Errorf("failed to get values map data, err: %s", err) + return nil, err + } + + currentValuesYaml, err := yaml.Marshal(valuesMap) + if err != nil { + return nil, err + } + + return &ValuesResp{ValuesYaml: string(currentValuesYaml)}, nil +} + +func ListReleases(args *HelmReleaseQueryArgs, envName string, log *zap.SugaredLogger) ([]*HelmReleaseResp, error) { + projectName, filterStr := args.ProjectName, args.Filter + opt := &commonrepo.ProductFindOptions{Name: projectName, EnvName: envName} + prod, err := commonrepo.NewProductColl().Find(opt) + if err != nil { + return nil, fmt.Errorf("failed to find project: %s, err: %s", projectName, err) + } + + restConfig, err := kube.GetRESTConfig(prod.ClusterID) + if err != nil { + log.Errorf("GetRESTConfig error: %s", err) + return nil, fmt.Errorf("failed to get k8s rest config, err: %s", err) + } + helmClientInterface, err := helmtool.NewClientFromRestConf(restConfig, prod.Namespace) + if err != nil { + log.Errorf("[%s][%s] NewClientFromRestConf error: %s", envName, projectName, err) + return nil, fmt.Errorf("failed to init helm client, err: %s", err) + } + + filter := &ReleaseFilter{} + if len(filterStr) > 0 { + data, err := jsonutil.ToJSON(filterStr) + if err != nil { + log.Warnf("invalid filter, err: %s", err) + return nil, fmt.Errorf("invalid filter, err: %s", err) + } + + if err = json.Unmarshal(data, filter); err != nil { + log.Warnf("Invalid filter, err: %s", err) + return nil, fmt.Errorf("invalid filter, err: %s", err) + } + } + + releases, err := listReleaseInNamespace(helmClientInterface, filter) + if err != nil { + return nil, fmt.Errorf("failed to list release, err: %s", err) + } + releaseMap := make(map[string]*release.Release) + for _, re := range releases { + releaseMap[re.Name] = re + } + + releaseNameMap, err := commonservice.GetServiceNameToReleaseNameMap(prod) + if err != nil { + return nil, fmt.Errorf("failed to build release-service map: %s", err) + } + + svcDatSetMap := make(map[string]*SvcDataSet) + svcDataList := make([]*SvcDataSet, 0) + + for _, svcGroup := range prod.Services { + for _, prodSvc := range svcGroup { + serviceName := prodSvc.ServiceName + releaseName := releaseNameMap[serviceName] + svcDataSet := &SvcDataSet{ + ProdSvc: prodSvc, + SvcRelease: releaseMap[releaseName], + } + svcDatSetMap[serviceName] = svcDataSet + svcDataList = append(svcDataList, svcDataSet) + } + } + + // set service template data + serviceTmpls, err := commonrepo.NewServiceColl().ListMaxRevisionsByProduct(projectName) + if err != nil { + return nil, errors.Errorf("failed to list service templates for project: %s", projectName) + } + for _, tmplSvc := range serviceTmpls { + if svcDataSet, ok := svcDatSetMap[tmplSvc.ServiceName]; ok { + svcDataSet.TmplSvc = tmplSvc + } + } + + filterFunc := func(svcDataSet *SvcDataSet) bool { + // filter by name + if filter.Name != "" && !strings.Contains(svcDataSet.ProdSvc.ServiceName, filter.Name) { + return false + } + // filter by status + switch filter.Status { + case HelmReleaseStatusDeployed: + if svcDataSet.SvcRelease == nil || svcDataSet.SvcRelease.Info.Status != release.StatusDeployed { + return false + } + case HelmReleaseStatusFailed: + if svcDataSet.SvcRelease != nil && svcDataSet.SvcRelease.Info.Status != release.StatusFailed && svcDataSet.SvcRelease.Info.Status != release.StatusSuperseded { + return false + } + if svcDataSet.ProdSvc.Error == "" { + return false + } + case HelmReleaseStatusPending: + if svcDataSet.SvcRelease == nil || (!svcDataSet.SvcRelease.Info.Status.IsPending() && svcDataSet.SvcRelease.Info.Status != release.StatusUninstalling) { + return false + } + case HelmReleaseStatusNotDeployed: + if svcDataSet.SvcRelease != nil { + return false + } + } + return true + } + + ret := make([]*HelmReleaseResp, 0) + + for _, svcDataSet := range svcDataList { + if !filterFunc(svcDataSet) { + continue + } + + updatable := false + if svcDataSet.TmplSvc != nil { + if svcDataSet.ProdSvc.Revision != svcDataSet.TmplSvc.Revision { + updatable = true + } + } + + respObj := &HelmReleaseResp{ + ServiceName: svcDataSet.ProdSvc.ServiceName, + Status: HelmReleaseStatusNotDeployed, + Updatable: updatable, + Error: svcDataSet.ProdSvc.Error, + } + + if svcDataSet.SvcRelease != nil { + respObj.ReleaseName = svcDataSet.SvcRelease.Name + respObj.Revision = svcDataSet.SvcRelease.Version + respObj.Chart = svcDataSet.SvcRelease.Chart.Name() + respObj.AppVersion = svcDataSet.SvcRelease.Chart.AppVersion() + respObj.Status = getReleaseStatus(svcDataSet.SvcRelease) + } + + if svcDataSet.ProdSvc.Error != "" { + respObj.Status = HelmReleaseStatusFailed + } + + ret = append(ret, respObj) + } + + return ret, nil +} + +func loadChartFilesInfo(productName, serviceName string, revision int64, dir string) ([]*types.FileInfo, error) { + base := config.LocalServicePathWithRevision(productName, serviceName, revision) + + var fis []*types.FileInfo + files, err := os.ReadDir(filepath.Join(base, serviceName, dir)) + if err != nil { + log.Warnf("failed to read chart info for service %s with revision %d", serviceName, revision) + base = config.LocalServicePath(productName, serviceName) + files, err = os.ReadDir(filepath.Join(base, serviceName, dir)) + if err != nil { + return nil, err + } + } + + for _, file := range files { + info, _ := file.Info() + if info == nil { + continue + } + fi := &types.FileInfo{ + Parent: dir, + Name: file.Name(), + Size: info.Size(), + Mode: file.Type(), + ModTime: info.ModTime().Unix(), + IsDir: file.IsDir(), + } + + fis = append(fis, fi) + } + return fis, nil +} + +//prepare chart version data +func prepareChartVersionData(prod *models.Product, serviceObj *models.Service) error { + productName := prod.ProductName + serviceName, revision := serviceObj.ServiceName, serviceObj.Revision + base := config.LocalServicePathWithRevision(productName, serviceName, revision) + if err := commonservice.PreloadServiceManifestsByRevision(base, serviceObj); err != nil { + log.Warnf("failed to get chart of revision: %d for service: %s, use latest version", revision, serviceName) + // use the latest version when it fails to download the specific version + base = config.LocalServicePath(productName, serviceName) + if err = commonservice.PreLoadServiceManifests(base, serviceObj); err != nil { + log.Errorf("failed to load chart info for service %v", serviceObj.ServiceName) + return err + } + } + + fullPath := filepath.Join(base, serviceObj.ServiceName) + deliveryChartPath := filepath.Join(config.LocalDeliveryChartPathWithRevision(productName, serviceObj.ServiceName, serviceObj.Revision), serviceObj.ServiceName) + err := copy.Copy(fullPath, deliveryChartPath) + if err != nil { + return err + } + + restConfig, err := kube.GetRESTConfig(prod.ClusterID) + if err != nil { + log.Errorf("get rest config error: %s", err) + return err + } + helmClient, err := helmtool.NewClientFromRestConf(restConfig, prod.Namespace) + if err != nil { + log.Errorf("[%s][%s] init helm client error: %s", prod.EnvName, productName, err) + return err + } + + releaseName := util.GeneReleaseName(serviceObj.GetReleaseNaming(), prod.ProductName, prod.Namespace, prod.EnvName, serviceObj.ServiceName) + valuesMap, err := helmClient.GetReleaseValues(releaseName, true) + if err != nil { + log.Errorf("failed to get values map data, err: %s", err) + return err + } + + currentValuesYaml, err := yaml.Marshal(valuesMap) + if err != nil { + return err + } + + // write values.yaml + if err = os.WriteFile(filepath.Join(deliveryChartPath, setting.ValuesYaml), currentValuesYaml, 0644); err != nil { + return err + } + + return nil +} + +func GetChartInfos(productName, envName, serviceName string, log *zap.SugaredLogger) (*HelmChartsResp, error) { + opt := &commonrepo.ProductFindOptions{Name: productName, EnvName: envName} + prod, err := commonrepo.NewProductColl().Find(opt) + if err != nil { + return nil, e.ErrGetHelmCharts.AddErr(err) + } + renderSet, err := FindHelmRenderSet(productName, prod.Render.Name, prod.EnvName, log) + if err != nil { + log.Errorf("[%s][P:%s] find product renderset error: %v", envName, productName, err) + return nil, e.ErrGetHelmCharts.AddErr(err) + } + + chartMap := make(map[string]*template.RenderChart) + for _, chart := range renderSet.ChartInfos { + chartMap[chart.ServiceName] = chart + } + + allServiceMap := prod.GetServiceMap() + serviceMap := make(map[string]*models.ProductService) + + //validate data, make sure service and chart info exists + if len(serviceName) > 0 { + serviceList := strings.Split(serviceName, ",") + for _, singleService := range serviceList { + if service, ok := allServiceMap[singleService]; ok { + serviceMap[service.ServiceName] = service + } else { + return nil, e.ErrGetHelmCharts.AddDesc(fmt.Sprintf("failed to find service %s in target namespace", singleService)) + } + } + } else { + serviceMap = allServiceMap + } + + if len(serviceMap) == 0 { + return nil, nil + } + + ret := &HelmChartsResp{ + ChartInfos: make([]*ChartInfo, 0), + FileInfos: make([]*types.FileInfo, 0), + } + + errList := new(multierror.Error) + wg := sync.WaitGroup{} + + for _, service := range serviceMap { + ret.ChartInfos = append(ret.ChartInfos, &ChartInfo{ + ServiceName: service.ServiceName, + Revision: service.Revision, + }) + wg.Add(1) + // download chart info with particular version + go func(serviceName string, revision int64) { + defer wg.Done() + serviceObj, err := commonrepo.NewServiceColl().Find(&commonrepo.ServiceFindOption{ + ProductName: productName, + ServiceName: serviceName, + Revision: revision, + Type: setting.HelmDeployType, + }) + if err != nil { + log.Errorf("failed to query services name: %s, revision: %d, error: %s", serviceName, revision, err) + errList = multierror.Append(errList, fmt.Errorf("failed to query service, serviceName: %s, revision: %d", serviceName, revision)) + return + } + err = prepareChartVersionData(prod, serviceObj) + if err != nil { + errList = multierror.Append(errList, fmt.Errorf("failed to prepare chart info for service %s", serviceObj.ServiceName)) + return + } + }(service.ServiceName, service.Revision) + } + wg.Wait() + + if errList.ErrorOrNil() != nil { + return nil, errList.ErrorOrNil() + } + + // expand file info for first service + serviceToExpand := ret.ChartInfos[0].ServiceName + fis, err := loadChartFilesInfo(productName, serviceToExpand, serviceMap[serviceToExpand].Revision, "") + if err != nil { + log.Errorf("Failed to load service file info, err: %s", err) + return nil, e.ErrListTemplate.AddErr(err) + } + ret.FileInfos = fis + + return ret, nil +} + +func GetImageInfos(productName, envName, serviceNames string, log *zap.SugaredLogger) (*ChartImagesResp, error) { + opt := &commonrepo.ProductFindOptions{Name: productName, EnvName: envName} + prod, err := commonrepo.NewProductColl().Find(opt) + if err != nil { + return nil, fmt.Errorf("failed to find product: %s:%s to get image infos, err: %s", productName, envName, err) + } + + restConfig, err := kube.GetRESTConfig(prod.ClusterID) + if err != nil { + return nil, fmt.Errorf("failed to get rest config: %s:%s to get image infos, err: %s", productName, envName, err) + } + helmClient, err := helmtool.NewClientFromRestConf(restConfig, prod.Namespace) + if err != nil { + return nil, fmt.Errorf("failed to init kube client: %s:%s to get image infos, err: %s", productName, envName, err) + } + + // filter releases, only list releases deployed by zadig + serviceMap := prod.GetServiceMap() + templateSvcs, err := commonservice.GetProductUsedTemplateSvcs(prod) + if err != nil { + return nil, fmt.Errorf("failed to get service tempaltes, err: %s", err) + } + templateSvcMap := make(map[string]*models.Service) + for _, ts := range templateSvcs { + templateSvcMap[ts.ServiceName] = ts + } + services := strings.Split(serviceNames, ",") + + ret := &ChartImagesResp{} + + for _, svcName := range services { + prodSvc, ok := serviceMap[svcName] + if !ok || prodSvc == nil { + return nil, fmt.Errorf("failed to find service: %s in product", svcName) + } + + ts, ok := templateSvcMap[svcName] + if !ok { + return nil, fmt.Errorf("failed to find template service: %s", svcName) + } + + releaseName := util.GeneReleaseName(ts.GetReleaseNaming(), productName, prod.Namespace, prod.EnvName, svcName) + valuesYaml, err := helmClient.GetReleaseValues(releaseName, true) + if err != nil { + return nil, fmt.Errorf("failed to get values for relase: %s, err: %s", releaseName, err) + } + + flatMap, err := converter.Flatten(valuesYaml) + if err != nil { + return nil, fmt.Errorf("failed to get flat map url for release :%s", releaseName) + } + + svcImage := &ServiceImages{ + ServiceName: svcName, + Images: nil, + } + + allModules, err := commonrepo.NewBuildColl().List(&commonrepo.BuildListOption{ProductName: productName}) + if err != nil { + return nil, fmt.Errorf("failed to list builds for project: %s, err: %s", productName, err) + } + + containerNameSet := sets.NewString() + for _, build := range allModules { + for _, target := range build.Targets { + if target.ServiceName == svcName { + containerNameSet.Insert(target.ServiceModule) + } + } + } + + for _, container := range prodSvc.Containers { + if container.ImagePath == nil { + return nil, fmt.Errorf("failed to parse image for container:%s", container.Image) + } + + imageSearchRule := &template.ImageSearchingRule{ + Repo: container.ImagePath.Repo, + Image: container.ImagePath.Image, + Tag: container.ImagePath.Tag, + } + pattern := imageSearchRule.GetSearchingPattern() + imageUrl, err := commonservice.GeneImageURI(pattern, flatMap) + if err != nil { + return nil, fmt.Errorf("failed to get image url for container:%s", container.Image) + } + + svcImage.Images = append(svcImage.Images, &ImageData{ + util.GetImageNameFromContainerInfo(container.ImageName, container.Name), + commonservice.ExtractImageTag(imageUrl), + containerNameSet.Has(container.ImageName), + }) + } + ret.ServiceImages = append(ret.ServiceImages, svcImage) + } + return ret, nil +} diff --git a/pkg/microservice/aslan/core/environment/service/ide.go b/pkg/microservice/aslan/core/environment/service/ide.go new file mode 100644 index 0000000000000000000000000000000000000000..6cbfcc6d7bb4b89058c9f2d2a89dc079b8b4e294 --- /dev/null +++ b/pkg/microservice/aslan/core/environment/service/ide.go @@ -0,0 +1,535 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "go.uber.org/zap" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types" + "github.com/koderover/zadig/pkg/util" +) + +var ErrUnsupportedService = errors.New("unsupported workload type (only support Deployment/StatefulSet) or workload number is not 1") + +func PatchWorkload(ctx context.Context, projectName, envName, serviceName, devImage string) (*types.WorkloadInfo, error) { + project, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: projectName, + EnvName: envName, + }) + if err != nil { + return nil, fmt.Errorf("failed to query env %q of project %q: %s", envName, projectName, err) + } + ns := project.Namespace + clusterID := project.ClusterID + + kclient, err := kubeclient.GetKubeClient(config.HubServerAddress(), clusterID) + if err != nil { + return nil, fmt.Errorf("failed to get kube client: %s", err) + } + + selector := labels.Set{setting.ProductLabel: projectName, setting.ServiceLabel: serviceName}.AsSelector() + workloadType, err := getWorkloadType(ctx, kclient, selector, ns) + if err != nil { + return nil, fmt.Errorf("failed to get workload type: %s", err) + } + log.Infof("workloadType: %s", workloadType) + + podName, err := patchWorkload(ctx, kclient, selector, ns, devImage, workloadType) + + return &types.WorkloadInfo{ + PodName: podName, + PodNamespace: ns, + }, err +} + +func RecoverWorkload(ctx context.Context, projectName, envName, serviceName string) error { + project, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: projectName, + EnvName: envName, + }) + if err != nil { + return fmt.Errorf("failed to query env %q of project %q: %s", envName, projectName, err) + } + ns := project.Namespace + clusterID := project.ClusterID + + kclient, err := kubeclient.GetKubeClient(config.HubServerAddress(), clusterID) + if err != nil { + return fmt.Errorf("failed to get kube client: %s", err) + } + + selector := labels.Set{setting.ProductLabel: projectName, setting.ServiceLabel: serviceName}.AsSelector() + workloadType, err := getWorkloadType(ctx, kclient, selector, ns) + if err != nil { + return fmt.Errorf("failed to get workload type: %s", err) + } + log.Infof("workloadType: %s", workloadType) + + return recoverWorkload(ctx, kclient, selector, ns, workloadType) +} + +func scaleAndWaitService(ctx context.Context, envName, projectName, serviceName, ns string, workloadType types.WorkloadType, kclient client.Client, selector labels.Selector) error { + logger, err := zap.NewProduction() + if err != nil { + return err + } + + err = ScaleService(envName, projectName, serviceName, 1, logger.Sugar()) + if err != nil { + return err + } + + return wait.PollImmediate(2*time.Second, 3*time.Minute, func() (done bool, err error) { + switch workloadType { + case types.DeploymentWorkload: + objs := &appsv1.DeploymentList{} + err := kclient.List(ctx, objs, &client.ListOptions{ + Namespace: ns, + LabelSelector: selector, + }) + if err != nil { + // Don't return error to continue polling. + log.Warnf("Failed to list deployments: %s", err) + return false, nil + } + + if len(objs.Items) != 1 { + return false, fmt.Errorf("invalid number of deployments: %d", len(objs.Items)) + } + + obj := objs.Items[0] + if obj.Status.Replicas == 1 && obj.Status.ReadyReplicas == 1 { + return true, nil + } + + return false, nil + case types.StatefulSetWorkload: + objs := &appsv1.StatefulSetList{} + err := kclient.List(ctx, objs, &client.ListOptions{ + Namespace: ns, + LabelSelector: selector, + }) + if err != nil { + // Don't return error to continue polling. + log.Warnf("Failed to list statefulsets: %s", err) + return false, nil + } + + if len(objs.Items) != 1 { + return false, fmt.Errorf("invalid number of statefulsets: %d", len(objs.Items)) + } + + obj := objs.Items[0] + if obj.Status.Replicas == 1 && obj.Status.ReadyReplicas == 1 { + return true, nil + } + + return false, nil + default: + return false, fmt.Errorf("unsupported workload type: %s", workloadType) + } + }) +} + +func getWorkloadType(ctx context.Context, kclient client.Client, selector labels.Selector, ns string) (types.WorkloadType, error) { + res := []types.WorkloadType{} + + deploymentList := &appsv1.DeploymentList{} + err := kclient.List(ctx, deploymentList, &client.ListOptions{ + Namespace: ns, + LabelSelector: selector, + }) + if err != nil { + return "", fmt.Errorf("failed to list deployment: %s", err) + } + if len(deploymentList.Items) == 1 { + res = append(res, types.DeploymentWorkload) + } + + stsList := &appsv1.StatefulSetList{} + err = kclient.List(ctx, stsList, &client.ListOptions{ + Namespace: ns, + LabelSelector: selector, + }) + if err != nil { + return "", fmt.Errorf("failed to list statefulset: %s", err) + } + if len(stsList.Items) == 1 { + res = append(res, types.StatefulSetWorkload) + } + + if len(res) != 1 { + return "", ErrUnsupportedService + } + + return res[0], nil +} + +func patchWorkload(ctx context.Context, kclient client.Client, selector labels.Selector, ns, devImage string, workloadType types.WorkloadType) (string, error) { + switch workloadType { + case types.DeploymentWorkload: + err := patchDeployment(ctx, kclient, selector, ns, devImage) + if err != nil { + return "", err + } + case types.StatefulSetWorkload: + err := patchStatefulSet(ctx, kclient, selector, ns, devImage) + if err != nil { + return "", err + } + default: + return "", fmt.Errorf("unsupported workload type: %s", workloadType) + } + + err := waitPodReady(ctx, kclient, selector, ns) + if err != nil { + return "", err + } + + pod, err := getPod(ctx, kclient, selector, ns) + if err != nil { + return "", fmt.Errorf("failed to get pod: %s", err) + } + + return pod.Name, nil +} + +func patchDeployment(ctx context.Context, kclient client.Client, selector labels.Selector, ns, devImage string) error { + objs := &appsv1.DeploymentList{} + err := kclient.List(ctx, objs, &client.ListOptions{ + Namespace: ns, + LabelSelector: selector, + }) + if err != nil { + return fmt.Errorf("failed to list deployments: %s", err) + } + if len(objs.Items) != 1 { + return fmt.Errorf("invalid number of deployments: %d", len(objs.Items)) + } + + deployment := objs.Items[0] + + spec, err := json.Marshal(deployment.Spec) + if err != nil { + return fmt.Errorf("failed to marshal deployment spec: %s", err) + } + if deployment.Annotations == nil { + deployment.Annotations = map[string]string{} + } + deployment.Annotations[types.OriginSpec] = string(spec) + + deployment.Spec.Replicas = util.GetInt32Pointer(1) + deployment.Spec.Template.Spec.Volumes = []corev1.Volume{ + corev1.Volume{ + Name: "syncthing", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + } + deployment.Spec.Template.Spec.Containers = []corev1.Container{ + corev1.Container{ + Name: types.IDEContainerNameDev, + Image: devImage, + ImagePullPolicy: corev1.PullAlways, + Command: []string{"/bin/sh", "-c", "tail -f /dev/null"}, + WorkingDir: types.DevmodeWorkDir, + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + VolumeMounts: []corev1.VolumeMount{ + corev1.VolumeMount{ + Name: "syncthing", + MountPath: types.DevmodeWorkDir, + }, + }, + }, + corev1.Container{ + Name: types.IDEContainerNameSidecar, + Image: types.IDESidecarImage, + ImagePullPolicy: corev1.PullAlways, + Command: []string{"/bin/sh", "-c", "tail -f /dev/null"}, + WorkingDir: types.DevmodeWorkDir, + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("300m"), + corev1.ResourceMemory: resource.MustParse("300Mi"), + }, + }, + VolumeMounts: []corev1.VolumeMount{ + corev1.VolumeMount{ + Name: "syncthing", + MountPath: types.DevmodeWorkDir, + }, + }, + }, + } + + return kclient.Update(ctx, &deployment) +} + +func waitPodReady(ctx context.Context, kclient client.Client, selector labels.Selector, ns string) error { + return wait.PollImmediate(2*time.Second, 3*time.Minute, func() (done bool, err error) { + objs := &corev1.PodList{} + err = kclient.List(ctx, objs, &client.ListOptions{ + Namespace: ns, + LabelSelector: selector, + }) + if err != nil { + log.Warnf("failed to list pod: %s", err) + return false, nil + } + + for _, pod := range objs.Items { + if len(pod.Spec.Containers) != 2 { + continue + } + + found := true + for _, container := range pod.Spec.Containers { + if container.Name == types.IDEContainerNameDev || container.Name == types.IDEContainerNameSidecar { + continue + } + + found = false + } + + if found && pod.Status.Phase == corev1.PodRunning { + return true, nil + } + } + + return false, nil + }) +} + +func patchStatefulSet(ctx context.Context, kclient client.Client, selector labels.Selector, ns, devImage string) error { + objs := &appsv1.StatefulSetList{} + err := kclient.List(ctx, objs, &client.ListOptions{ + Namespace: ns, + LabelSelector: selector, + }) + if err != nil { + return fmt.Errorf("failed to list statefulsets: %s", err) + } + if len(objs.Items) != 1 { + return fmt.Errorf("invalid number of statefulsets: %d", len(objs.Items)) + } + + sts := objs.Items[0] + + spec, err := json.Marshal(sts.Spec) + if err != nil { + return fmt.Errorf("failed to marshal statefulset spec: %s", err) + } + if sts.Annotations == nil { + sts.Annotations = map[string]string{} + } + sts.Annotations[types.OriginSpec] = string(spec) + + sts.Spec.Replicas = util.GetInt32Pointer(1) + sts.Spec.Template.Spec.Volumes = []corev1.Volume{ + corev1.Volume{ + Name: "syncthing", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + } + sts.Spec.Template.Spec.Containers = []corev1.Container{ + corev1.Container{ + Name: types.IDEContainerNameDev, + Image: devImage, + ImagePullPolicy: corev1.PullAlways, + Command: []string{"/bin/sh", "-c", "tail -f /dev/null"}, + WorkingDir: types.DevmodeWorkDir, + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + VolumeMounts: []corev1.VolumeMount{ + corev1.VolumeMount{ + Name: "syncthing", + MountPath: types.DevmodeWorkDir, + }, + }, + }, + corev1.Container{ + Name: types.IDEContainerNameSidecar, + Image: types.IDESidecarImage, + ImagePullPolicy: corev1.PullAlways, + Command: []string{"/bin/sh", "-c", "tail -f /dev/null"}, + WorkingDir: types.DevmodeWorkDir, + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("300m"), + corev1.ResourceMemory: resource.MustParse("300Mi"), + }, + }, + VolumeMounts: []corev1.VolumeMount{ + corev1.VolumeMount{ + Name: "syncthing", + MountPath: types.DevmodeWorkDir, + }, + }, + }, + } + + return kclient.Update(ctx, &sts) +} + +func recoverWorkload(ctx context.Context, kclient client.Client, selector labels.Selector, ns string, workloadType types.WorkloadType) error { + switch workloadType { + case types.DeploymentWorkload: + err := recoverDeployment(ctx, kclient, selector, ns) + if err != nil { + return err + } + case types.StatefulSetWorkload: + err := recoverStatefulSet(ctx, kclient, selector, ns) + if err != nil { + return err + } + default: + return fmt.Errorf("unsupported workload type: %s", workloadType) + } + + return waitPodReady(ctx, kclient, selector, ns) +} + +func recoverDeployment(ctx context.Context, kclient client.Client, selector labels.Selector, ns string) error { + objs := &appsv1.DeploymentList{} + err := kclient.List(ctx, objs, &client.ListOptions{ + Namespace: ns, + LabelSelector: selector, + }) + if err != nil { + return fmt.Errorf("failed to list deployments: %s", err) + } + if len(objs.Items) != 1 { + return fmt.Errorf("invalid number of deployments: %d", len(objs.Items)) + } + + deployment := objs.Items[0] + if deployment.Annotations == nil || deployment.Annotations[types.OriginSpec] == "" { + return fmt.Errorf("cant' find original workload spec from annotation %q", types.OriginSpec) + } + + var spec appsv1.DeploymentSpec + if err := json.Unmarshal([]byte(deployment.Annotations[types.OriginSpec]), &spec); err != nil { + return fmt.Errorf("failed to unmarshal original workload spec from annotation: %s", err) + } + + deployment.Spec = spec + delete(deployment.Annotations, types.OriginSpec) + + return kclient.Update(ctx, &deployment) +} + +func recoverStatefulSet(ctx context.Context, kclient client.Client, selector labels.Selector, ns string) error { + objs := &appsv1.StatefulSetList{} + err := kclient.List(ctx, objs, &client.ListOptions{ + Namespace: ns, + LabelSelector: selector, + }) + if err != nil { + return fmt.Errorf("failed to list statefulsets: %s", err) + } + if len(objs.Items) != 1 { + return fmt.Errorf("invalid number of statefulsets: %d", len(objs.Items)) + } + + sts := objs.Items[0] + if sts.Annotations == nil || sts.Annotations[types.OriginSpec] == "" { + return fmt.Errorf("cant' find original workload spec from annotation %q", types.OriginSpec) + } + + var spec appsv1.StatefulSetSpec + if err := json.Unmarshal([]byte(sts.Annotations[types.OriginSpec]), &spec); err != nil { + return fmt.Errorf("failed to unmarshal original workload spec from annotation: %s", err) + } + + sts.Spec = spec + delete(sts.Annotations, types.OriginSpec) + + return kclient.Update(ctx, &sts) +} + +func getPod(ctx context.Context, kclient client.Client, selector labels.Selector, ns string) (*corev1.Pod, error) { + var targetPod *corev1.Pod + err := wait.PollImmediate(2*time.Second, 6*time.Second, func() (done bool, err error) { + objs := &corev1.PodList{} + err = kclient.List(ctx, objs, &client.ListOptions{ + Namespace: ns, + LabelSelector: selector, + }) + if err != nil { + log.Warnf("failed to list pod: %s", err) + return false, nil + } + + for _, pod := range objs.Items { + if len(pod.Spec.Containers) != 2 { + continue + } + + found := true + for _, container := range pod.Spec.Containers { + if container.Name == types.IDEContainerNameDev || container.Name == types.IDEContainerNameSidecar { + continue + } + + found = false + } + + if found { + targetPod = &pod + return true, nil + } + } + + return false, nil + }) + + if targetPod == nil { + return nil, fmt.Errorf("failed to find target pod: %s", err) + } + + return targetPod, nil +} diff --git a/pkg/microservice/aslan/core/environment/service/image.go b/pkg/microservice/aslan/core/environment/service/image.go index 34589fe2494ef2d73693ef1b61c34c0b0c4bf369..e623af0b689a43db4ddf4abe7ca196e33c46a0a9 100644 --- a/pkg/microservice/aslan/core/environment/service/image.go +++ b/pkg/microservice/aslan/core/environment/service/image.go @@ -17,35 +17,25 @@ limitations under the License. package service import ( - "bytes" "fmt" - "regexp" - "strings" "time" - helmclient "github.com/mittwald/go-helm-client" "go.uber.org/zap" - "helm.sh/helm/v3/pkg/strvals" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/yaml" + "github.com/koderover/zadig/pkg/microservice/aslan/config" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" templatemodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" e "github.com/koderover/zadig/pkg/tool/errors" helmtool "github.com/koderover/zadig/pkg/tool/helmclient" - "github.com/koderover/zadig/pkg/tool/kube/getter" "github.com/koderover/zadig/pkg/tool/kube/updater" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/util" - "github.com/koderover/zadig/pkg/util/converter" - yamlutil "github.com/koderover/zadig/pkg/util/yaml" ) type UpdateContainerImageArgs struct { @@ -58,37 +48,6 @@ type UpdateContainerImageArgs struct { Image string `json:"image"` } -const ( - imageUrlParseRegexString = `(?P.+/)?(?P[^:]+){1}(:)?(?P.+)?` -) - -var ( - imageParseRegex = regexp.MustCompile(imageUrlParseRegexString) -) - -func getHelmServiceName(namespace, resType, resName string, kubeClient client.Client) (string, error) { - res := &unstructured.Unstructured{} - res.SetGroupVersionKind(schema.GroupVersionKind{ - Group: "apps", - Version: "v1", - Kind: resType, - }) - found, err := getter.GetResourceInCache(namespace, resName, res, kubeClient) - if err != nil { - return "", fmt.Errorf("failed to find resource %s, type %s, err %s", resName, resType, err.Error()) - } - if !found { - return "", fmt.Errorf("failed to find resource %s, type %s", resName, resType) - } - annotation := res.GetAnnotations() - if len(annotation) > 0 { - if chartRelease, ok := annotation[setting.HelmReleaseNameAnnotation]; ok { - return util.ExtraServiceName(chartRelease, namespace), nil - } - } - return "", fmt.Errorf("failed to get annotation from resource %s, type %s", resName, resType) -} - func getValidMatchData(spec *models.ImagePathSpec) map[string]string { ret := make(map[string]string) if spec.Repo != "" { @@ -103,85 +62,12 @@ func getValidMatchData(spec *models.ImagePathSpec) map[string]string { return ret } -// parse image url to map: repo=>xxx/xx/xx image=>xx tag=>xxx -func resolveImageUrl(imageUrl string) map[string]string { - subMatchAll := imageParseRegex.FindStringSubmatch(imageUrl) - result := make(map[string]string) - exNames := imageParseRegex.SubexpNames() - for i, matchedStr := range subMatchAll { - if i != 0 && matchedStr != "" && matchedStr != ":" { - result[exNames[i]] = matchedStr - } - } - return result -} - -// replace image defines in yaml by new version -func replaceImage(sourceYaml string, imageValuesMap map[string]interface{}) (string, error) { - nestedMap, err := converter.Expand(imageValuesMap) - if err != nil { - return "", err - } - bs, err := yaml.Marshal(nestedMap) - if err != nil { - return "", err - } - mergedBs, err := yamlutil.Merge([][]byte{[]byte(sourceYaml), bs}) - if err != nil { - return "", err - } - return string(mergedBs), nil -} - -// AssignImageData assign image url data into match data -// matchData: image=>absolute-path repo=>absolute-path tag=>absolute-path -// return: absolute-image-path=>image-value absolute-repo-path=>repo-value absolute-tag-path=>tag-value -func assignImageData(imageUrl string, matchData map[string]string) (map[string]interface{}, error) { - ret := make(map[string]interface{}) - // total image url assigned into one single value - if len(matchData) == 1 { - for _, v := range matchData { - ret[v] = imageUrl - } - return ret, nil - } - - resolvedImageUrl := resolveImageUrl(imageUrl) - - // image url assigned into repo/image+tag - if len(matchData) == 3 { - ret[matchData[setting.PathSearchComponentRepo]] = strings.TrimSuffix(resolvedImageUrl[setting.PathSearchComponentRepo], "/") - ret[matchData[setting.PathSearchComponentImage]] = resolvedImageUrl[setting.PathSearchComponentImage] - ret[matchData[setting.PathSearchComponentTag]] = resolvedImageUrl[setting.PathSearchComponentTag] - return ret, nil - } - - if len(matchData) == 2 { - // image url assigned into repo/image + tag - if tagPath, ok := matchData[setting.PathSearchComponentTag]; ok { - ret[tagPath] = resolvedImageUrl[setting.PathSearchComponentTag] - for k, imagePath := range matchData { - if k == setting.PathSearchComponentTag { - continue - } - ret[imagePath] = fmt.Sprintf("%s%s", resolvedImageUrl[setting.PathSearchComponentRepo], resolvedImageUrl[setting.PathSearchComponentImage]) - break - } - return ret, nil - } - // image url assigned into repo + image(tag) - ret[matchData[setting.PathSearchComponentRepo]] = strings.TrimSuffix(resolvedImageUrl[setting.PathSearchComponentRepo], "/") - ret[matchData[setting.PathSearchComponentImage]] = fmt.Sprintf("%s:%s", resolvedImageUrl[setting.PathSearchComponentImage], resolvedImageUrl[setting.PathSearchComponentTag]) - return ret, nil - } - - return nil, fmt.Errorf("match data illegal, expect length: 1-3, actual length: %d", len(matchData)) -} - // prepare necessary data from db -func prepareData(namespace, serviceName string, containerName string, product *models.Product) (targetContainer *models.Container, +func prepareData(namespace, serviceName, containerName, image string, product *models.Product) (targetContainer *models.Container, targetChart *templatemodels.RenderChart, renderSet *models.RenderSet, serviceObj *models.Service, err error) { + imageName := commonservice.ExtractImageName(image) + var targetProductService *models.ProductService for _, productService := range product.GetServiceMap() { if productService.ServiceName != serviceName { @@ -189,20 +75,24 @@ func prepareData(namespace, serviceName string, containerName string, product *m } targetProductService = productService for _, container := range productService.Containers { - if container.Name != containerName { - continue + if container.ImageName == imageName { + targetContainer = container + break } - targetContainer = container - break } break } if targetContainer == nil { - err = fmt.Errorf("failed to find container %s", containerName) + err = fmt.Errorf("failed to find image config: %s for container %s", imageName, containerName) return } - renderSet, err = commonrepo.NewRenderSetColl().Find(&commonrepo.RenderSetFindOption{Name: namespace, Revision: product.Render.Revision}) + renderSet, err = commonrepo.NewRenderSetColl().Find(&commonrepo.RenderSetFindOption{ + ProductTmpl: product.ProductName, + Name: namespace, + EnvName: product.EnvName, + Revision: product.Render.Revision, + }) if err != nil { log.Errorf("[RenderSet.find] update product %s error: %s", product.ProductName, err.Error()) err = fmt.Errorf("failed to find redset name %s revision %d", namespace, product.Render.Revision) @@ -236,18 +126,17 @@ func prepareData(namespace, serviceName string, containerName string, product *m return } -func updateContainerForHelmChart(serviceName, resType, image, containerName string, product *models.Product) error { +func updateContainerForHelmChart(serviceName, resType, image, containerName string, product *models.Product, cl client.Client) error { var ( replaceValuesMap map[string]interface{} replacedValuesYaml string mergedValuesYaml string replacedMergedValuesYaml string - restConfig *rest.Config - helmClient helmclient.Client + helmClient *helmtool.HelmClient namespace string = product.Namespace ) - targetContainer, targetChart, renderSet, serviceObj, err := prepareData(namespace, serviceName, containerName, product) + targetContainer, targetChart, renderSet, serviceObj, err := prepareData(namespace, serviceName, containerName, image, product) if err != nil { return err } @@ -256,13 +145,13 @@ func updateContainerForHelmChart(serviceName, resType, image, containerName stri targetContainer.Image = image // prepare image replace info - replaceValuesMap, err = assignImageData(image, getValidMatchData(targetContainer.ImagePath)) + replaceValuesMap, err = commonservice.AssignImageData(image, getValidMatchData(targetContainer.ImagePath)) if err != nil { return fmt.Errorf("failed to pase image uri %s/%s, err %s", namespace, serviceName, err.Error()) } // replace image into service's values.yaml - replacedValuesYaml, err = replaceImage(targetChart.ValuesYaml, replaceValuesMap) + replacedValuesYaml, err = commonservice.ReplaceImage(targetChart.ValuesYaml, replaceValuesMap) if err != nil { return fmt.Errorf("failed to replace image uri %s/%s, err %s", namespace, serviceName, err.Error()) @@ -275,28 +164,33 @@ func updateContainerForHelmChart(serviceName, resType, image, containerName stri targetChart.ValuesYaml = replacedValuesYaml // merge override values and kvs into service's yaml - mergedValuesYaml, err = helmtool.MergeOverrideValues(replacedValuesYaml, targetChart.GetOverrideYaml(), targetChart.OverrideValues) + mergedValuesYaml, err = helmtool.MergeOverrideValues(replacedValuesYaml, renderSet.DefaultValues, targetChart.GetOverrideYaml(), targetChart.OverrideValues) if err != nil { return err } // replace image into final merged values.yaml - replacedMergedValuesYaml, err = replaceImage(mergedValuesYaml, replaceValuesMap) + replacedMergedValuesYaml, err = commonservice.ReplaceImage(mergedValuesYaml, replaceValuesMap) if err != nil { return err } - restConfig, err = kube.GetRESTConfig(product.ClusterID) + helmClient, err = helmtool.NewClientFromNamespace(product.ClusterID, namespace) if err != nil { return err } - helmClient, err = helmtool.NewClientFromRestConf(restConfig, namespace) - if err != nil { - return err + + param := &ReleaseInstallParam{ + ProductName: serviceObj.ProductName, + Namespace: namespace, + ReleaseName: util.GeneReleaseName(serviceObj.GetReleaseNaming(), serviceObj.ProductName, namespace, product.EnvName, serviceObj.ServiceName), + MergedValues: replacedMergedValuesYaml, + RenderChart: targetChart, + serviceObj: serviceObj, } // when replace image, should not wait - err = installOrUpgradeHelmChartWithValues(namespace, replacedMergedValuesYaml, targetChart, serviceObj, 0, helmClient) + err = installOrUpgradeHelmChartWithValues(param, false, helmClient) if err != nil { return err } @@ -321,10 +215,25 @@ func UpdateContainerImage(requestID string, args *UpdateContainerImageArgs, log } namespace := product.Namespace - kubeClient, err := kube.GetKubeClient(product.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), product.ClusterID) if err != nil { return e.ErrUpdateConainterImage.AddErr(err) } + // aws secrets needs to be refreshed + regs, err := commonservice.ListRegistryNamespaces("", true, log) + if err != nil { + log.Errorf("Failed to get registries to update container images, the error is: %s", err) + return err + } + for _, reg := range regs { + if reg.RegProvider == config.RegistryTypeAWS { + if err := kube.CreateOrUpdateRegistrySecret(namespace, reg, false, kubeClient); err != nil { + retErr := fmt.Errorf("failed to update pull secret for registry: %s, the error is: %s", reg.ID.Hex(), err) + log.Errorf("%s\n", retErr.Error()) + return retErr + } + } + } eventStart := time.Now().Unix() @@ -334,11 +243,11 @@ func UpdateContainerImage(requestID string, args *UpdateContainerImageArgs, log // update service in helm way if product.Source == setting.HelmDeployType { - serviceName, err := getHelmServiceName(namespace, args.Type, args.Name, kubeClient) + serviceName, err := commonservice.GetHelmServiceName(product, args.Type, args.Name, kubeClient) if err != nil { return e.ErrUpdateConainterImage.AddErr(err) } - err = updateContainerForHelmChart(serviceName, args.Type, args.Image, args.ContainerName, product) + err = updateContainerForHelmChart(serviceName, args.Type, args.Image, args.ContainerName, product, kubeClient) if err != nil { return e.ErrUpdateConainterImage.AddErr(err) } @@ -377,39 +286,3 @@ func UpdateContainerImage(requestID string, args *UpdateContainerImageArgs, log } return nil } - -func updateImageTagInValues(valuesYaml []byte, repo, newTag string) ([]byte, error) { - if !bytes.Contains(valuesYaml, []byte(repo)) { - return nil, nil - } - - valuesMap := map[string]interface{}{} - if err := yaml.Unmarshal(valuesYaml, &valuesMap); err != nil { - return nil, err - } - - valuesFlatMap, err := converter.Flatten(valuesMap) - if err != nil { - return nil, err - } - - var matchingKey string - for k, v := range valuesFlatMap { - if val, ok := v.(string); ok && repo == val { - matchingKey = k - } - } - if matchingKey == "" { - return nil, fmt.Errorf("key is not found for value %s in map %v", repo, valuesFlatMap) - } - - newKey := strings.Replace(matchingKey, ".repository", ".tag", 1) - if newKey == matchingKey { - return nil, fmt.Errorf("field repository is not found in key %s", matchingKey) - } - if err := strvals.ParseInto(fmt.Sprintf("%s=%s", newKey, newTag), valuesMap); err != nil { - return nil, err - } - - return yaml.Marshal(valuesMap) -} diff --git a/pkg/microservice/aslan/core/environment/service/ingress.go b/pkg/microservice/aslan/core/environment/service/ingress.go new file mode 100644 index 0000000000000000000000000000000000000000..ee9aab6ed3e3d7e0479fe3edf4c65239b241433b --- /dev/null +++ b/pkg/microservice/aslan/core/environment/service/ingress.go @@ -0,0 +1,249 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + + "go.uber.org/zap" + extensionsv1beta1 "k8s.io/api/extensions/v1beta1" + v1 "k8s.io/api/networking/v1" + "sigs.k8s.io/yaml" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/kube/getter" + "github.com/koderover/zadig/pkg/tool/kube/serializer" + "github.com/koderover/zadig/pkg/tool/kube/updater" +) + +type ListIngressesResponse struct { + *ResourceResponseBase + IngressName string `json:"ingress_name"` + HostInfo string `json:"host_info"` + Address string `json:"address"` + Ports string `json:"ports"` + ErrorReason string `json:"error_reason"` +} + +func ListIngresses(envName, productName string, log *zap.SugaredLogger) ([]*ListIngressesResponse, error) { + product, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: productName, + EnvName: envName, + }) + if err != nil { + return nil, e.ErrListResources.AddErr(err) + } + kubeCli, err := kubeclient.GetKubeClient(config.HubServerAddress(), product.ClusterID) + if err != nil { + return nil, e.ErrListResources.AddErr(err) + } + cliSet, err := kubeclient.GetKubeClientSet(config.HubServerAddress(), product.ClusterID) + if err != nil { + return nil, e.ErrListResources.AddErr(err) + } + version, err := cliSet.Discovery().ServerVersion() + if err != nil { + log.Errorf("Failed to get server version info for cluster: %s, the error is: %s", product.ClusterID, err) + return nil, err + } + ingresss, err := getter.ListIngresses(product.Namespace, kubeCli, kubeclient.VersionLessThan122(version)) + if err != nil { + log.Error(err) + return nil, e.ErrListResources.AddDesc(err.Error()) + } + + var res []*ListIngressesResponse + for _, ingress := range ingresss.Items { + ingress.SetManagedFields(nil) + ingress.SetResourceVersion("") + yamlData, err := yaml.Marshal(ingress.Object) + if err != nil { + log.Error(err) + return nil, e.ErrListResources.AddDesc(err.Error()) + } + + if ingress.GetAPIVersion() == v1.SchemeGroupVersion.String() { + itemJson, err := ingress.MarshalJSON() + if err != nil { + return nil, err + } + itemIngress := &v1.Ingress{} + err = json.Unmarshal(itemJson, itemIngress) + if err != nil { + return nil, err + } + hostInfo := "" + for _, rule := range itemIngress.Spec.Rules { + hostInfo += rule.Host + "," + } + hostInfo = strings.TrimSuffix(hostInfo, ",") + address, ports, errorReason := "", "", "" + for _, addr := range itemIngress.Status.LoadBalancer.Ingress { + address += addr.IP + "," + for _, portStatus := range addr.Ports { + if portStatus.Error != nil { + errorReason += *portStatus.Error + ";" + } else { + ports += string(portStatus.Port) + "," + } + } + } + address = strings.TrimSuffix(address, ",") + errorReason = strings.TrimSuffix(errorReason, ";") + ports = strings.TrimSuffix(ports, ",") + + resElem := &ListIngressesResponse{ + IngressName: ingress.GetName(), + HostInfo: hostInfo, + Ports: ports, + ErrorReason: errorReason, + Address: address, + ResourceResponseBase: &ResourceResponseBase{ + Name: ingress.GetName(), + Type: config.CommonEnvCfgTypeIngress, + EnvName: envName, + ProjectName: productName, + YamlData: string(yamlData), + CreateTime: ingress.GetCreationTimestamp().Time, + }, + } + resElem.setSourceDetailData(&ingress) + res = append(res, resElem) + } + + if ingress.GetAPIVersion() == extensionsv1beta1.SchemeGroupVersion.String() { + itemJson, err := ingress.MarshalJSON() + if err != nil { + return nil, err + } + itemIngress := &extensionsv1beta1.Ingress{} + err = json.Unmarshal(itemJson, itemIngress) + if err != nil { + return nil, err + } + hostInfo, ports := "", "" + for _, rule := range itemIngress.Spec.Rules { + hostInfo += rule.Host + "," + if rule.HTTP != nil { + for _, path := range rule.HTTP.Paths { + ports += path.Backend.ServicePort.String() + "," + } + } + } + hostInfo = strings.TrimSuffix(hostInfo, ",") + address, errorReason := "", "" + for _, addr := range itemIngress.Status.LoadBalancer.Ingress { + address += addr.IP + "," + for _, portStatus := range addr.Ports { + if portStatus.Error != nil { + errorReason += *portStatus.Error + ";" + } + } + } + address = strings.TrimSuffix(address, ",") + errorReason = strings.TrimSuffix(errorReason, ";") + ports = strings.TrimSuffix(ports, ",") + + resElem := &ListIngressesResponse{ + IngressName: ingress.GetName(), + HostInfo: hostInfo, + Ports: ports, + ErrorReason: errorReason, + Address: address, + ResourceResponseBase: &ResourceResponseBase{ + Name: ingress.GetName(), + Type: config.CommonEnvCfgTypeIngress, + EnvName: envName, + ProjectName: productName, + YamlData: string(yamlData), + CreateTime: ingress.GetCreationTimestamp().Time, + }, + } + resElem.setSourceDetailData(&ingress) + res = append(res, resElem) + } + } + sort.SliceStable(res, func(i, j int) bool { + return res[i].IngressName < res[j].IngressName + }) + return res, nil +} + +type UpdateIngressArgs struct { + EnvName string `json:"env_name"` + ProductName string `json:"product_name"` + IngressName string `json:"ingress_name"` + YamlData string `json:"yaml_data"` + RestartAssociatedSvc bool `json:"restart_associated_svc"` +} + +func UpdateOrCreateIngress(args *models.CreateUpdateCommonEnvCfgArgs, userName string, isCreate bool, log *zap.SugaredLogger) error { + u, err := serializer.NewDecoder().YamlToUnstructured([]byte(args.YamlData)) + if err != nil { + log.Errorf("Failed to convert yaml to Unstructured, manifest is\n%s\n, error: %v", args.YamlData, err) + return e.ErrUpdateResource.AddDesc("ingress Yaml Name is incorrect") + } + if !isCreate && u.GetName() != args.Name { + return e.ErrUpdateResource.AddDesc("ingress Yaml Name is incorrect") + } + + product, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: args.ProductName, + EnvName: args.EnvName, + }) + if err != nil { + return e.ErrUpdateResource.AddErr(err) + } + + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), product.ClusterID) + if err != nil { + return e.ErrUpdateResource.AddErr(err) + } + + yamlData, err := ensureLabelAndNs(u, product.Namespace, args.ProductName) + if err != nil { + return e.ErrUpdateResource.AddErr(err) + } + + err = updater.UpdateOrCreateUnstructured(u, kubeClient) + if err != nil { + log.Errorf("Failed to UpdateOrCreateIngress %s, manifest is\n%v\n, error: %v", u.GetKind(), u, err) + return e.ErrUpdateResource.AddErr(fmt.Errorf("Failed to UpdateOrCreateIngress %s, manifest is\n%v\n, error: %v", u.GetKind(), u, err)) + } + envIngress := &models.EnvResource{ + ProductName: args.ProductName, + UpdateUserName: userName, + EnvName: args.EnvName, + Namespace: product.Namespace, + Name: u.GetName(), + YamlData: yamlData, + Type: string(config.CommonEnvCfgTypeIngress), + SourceDetail: args.SourceDetail, + AutoSync: args.AutoSync, + } + if commonrepo.NewEnvResourceColl().Create(envIngress) != nil { + return e.ErrUpdateResource.AddDesc(err.Error()) + } + return err +} diff --git a/pkg/microservice/aslan/core/environment/service/k8s.go b/pkg/microservice/aslan/core/environment/service/k8s.go index 3f922b06ad1d2df98953311ec55ebb1d97cd7e82..d2f6dd72b9c3592377979306ac5608ce95008d43 100644 --- a/pkg/microservice/aslan/core/environment/service/k8s.go +++ b/pkg/microservice/aslan/core/environment/service/k8s.go @@ -25,18 +25,20 @@ import ( "github.com/hashicorp/go-multierror" "go.uber.org/zap" - "sigs.k8s.io/controller-runtime/pkg/client" - + versionedclient "istio.io/client-go/pkg/clientset/versioned" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/informers" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/kube/informer" ) type K8sService struct { @@ -48,11 +50,11 @@ type K8sService struct { // If service doesnt have pods, service status = success (all objects created) or failed (fail to create some objects). // 正常：StatusRunning or StatusSucceed // 错误：StatusError or StatusFailed -func (k *K8sService) queryServiceStatus(namespace, envName, productName string, serviceTmpl *commonmodels.Service, kubeClient client.Client) (string, string, []string) { +func (k *K8sService) queryServiceStatus(namespace, envName, productName string, serviceTmpl *commonmodels.Service, informer informers.SharedInformerFactory) (string, string, []string) { k.log.Infof("queryServiceStatus of service: %s of product: %s in namespace %s", serviceTmpl.ServiceName, productName, namespace) if len(serviceTmpl.Containers) > 0 { // 有容器时，根据pods status判断服务状态 - return queryPodsStatus(namespace, productName, serviceTmpl.ServiceName, kubeClient, k.log) + return queryPodsStatus(namespace, productName, serviceTmpl.ServiceName, informer, k.log) } return setting.PodSucceeded, setting.PodReady, []string{} @@ -84,11 +86,30 @@ func (k *K8sService) updateService(args *SvcOptArgs) error { return errors.New(e.UpsertServiceErrMsg) } - kubeClient, err := kube.GetKubeClient(exitedProd.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), exitedProd.ClusterID) + if err != nil { + return e.ErrUpdateEnv.AddErr(err) + } + + restConfig, err := kubeclient.GetRESTConfig(config.HubServerAddress(), exitedProd.ClusterID) + if err != nil { + return e.ErrUpdateEnv.AddErr(err) + } + + istioClient, err := versionedclient.NewForConfig(restConfig) if err != nil { return e.ErrUpdateEnv.AddErr(err) } + cls, err := kubeclient.GetKubeClientSet(config.HubServerAddress(), exitedProd.ClusterID) + if err != nil { + return e.ErrUpdateEnv.AddDesc(err.Error()) + } + inf, err := informer.NewInformer(exitedProd.ClusterID, exitedProd.Namespace, cls) + if err != nil { + return e.ErrUpdateEnv.AddDesc(err.Error()) + } + switch exitedProd.Status { case setting.ProductStatusCreating, setting.ProductStatusUpdating, setting.ProductStatusDeleting: k.log.Errorf("[%s][P:%s] Product is not in valid status", args.EnvName, args.ProductName) @@ -100,7 +121,7 @@ func (k *K8sService) updateService(args *SvcOptArgs) error { exitedProd.Render = &commonmodels.RenderInfo{ProductTmpl: exitedProd.ProductName} } // 检查renderset是否覆盖服务所有key - newRender, err := commonservice.ValidateRenderSet(args.ProductName, exitedProd.Render.Name, serviceInfo, k.log) + newRender, err := commonservice.ValidateRenderSet(args.ProductName, exitedProd.Render.Name, exitedProd.EnvName, serviceInfo, k.log) if err != nil { k.log.Errorf("[%s][P:%s] validate product renderset error: %v", args.EnvName, args.ProductName, err) return e.ErrUpdateProduct.AddDesc(err.Error()) @@ -112,7 +133,7 @@ func (k *K8sService) updateService(args *SvcOptArgs) error { exitedProd, svc, exitedProd.GetServiceMap()[svc.ServiceName], - newRender, kubeClient, k.log) + newRender, inf, kubeClient, istioClient, k.log) // 如果创建依赖服务组有返回错误, 停止等待 if err != nil { @@ -136,9 +157,7 @@ func (k *K8sService) updateService(args *SvcOptArgs) error { return nil } -// TODO: LOU: improve the status scope and definition, like pod status, service status, environment, cluster status, ... -// TODO: LOU: rewrite it -func (k *K8sService) listGroupServices(allServices []*commonmodels.ProductService, envName, productName string, kubeClient client.Client, productInfo *commonmodels.Product) []*commonservice.ServiceResp { +func (k *K8sService) listGroupServices(allServices []*commonmodels.ProductService, envName, productName string, informer informers.SharedInformerFactory, productInfo *commonmodels.Product) []*commonservice.ServiceResp { var wg sync.WaitGroup var resp []*commonservice.ServiceResp var mutex sync.RWMutex @@ -165,8 +184,8 @@ func (k *K8sService) listGroupServices(allServices []*commonmodels.ProductServic gp.ProductName = serviceTmpl.ProductName // 查询group下所有pods信息 - if kubeClient != nil { - gp.Status, gp.Ready, gp.Images = k.queryServiceStatus(productInfo.Namespace, envName, productName, serviceTmpl, kubeClient) + if informer != nil { + gp.Status, gp.Ready, gp.Images = k.queryServiceStatus(productInfo.Namespace, envName, productName, serviceTmpl, informer) // 如果产品正在创建中，且service status为ERROR（POD还没创建出来），则判断为Pending，尚未开始创建 if productInfo.Status == setting.ProductStatusCreating && gp.Status == setting.PodError { gp.Status = setting.PodPending @@ -191,7 +210,7 @@ func (k *K8sService) listGroupServices(allServices []*commonmodels.ProductServic return resp } -func (k *K8sService) createGroup(envName, productName, username string, group []*commonmodels.ProductService, renderSet *commonmodels.RenderSet, kubeClient client.Client) error { +func (k *K8sService) createGroup(envName, productName, username string, group []*commonmodels.ProductService, renderSet *commonmodels.RenderSet, informer informers.SharedInformerFactory, kubeClient client.Client) error { k.log.Infof("[Namespace:%s][Product:%s] createGroup", envName, productName) updatableServiceNameList := make([]string, 0) @@ -213,6 +232,16 @@ func (k *K8sService) createGroup(envName, productName, username string, group [] errList = multierror.Append(errList, err) } + restConfig, err := kubeclient.GetRESTConfig(config.HubServerAddress(), prod.ClusterID) + if err != nil { + return fmt.Errorf("failed to get rest config: %s", err) + } + + istioClient, err := versionedclient.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("failed to new istio client: %s", err) + } + var wg sync.WaitGroup var lock sync.Mutex var resources []*unstructured.Unstructured @@ -224,7 +253,7 @@ func (k *K8sService) createGroup(envName, productName, username string, group [] updatableServiceNameList = append(updatableServiceNameList, group[i].ServiceName) go func(svc *commonmodels.ProductService) { defer wg.Done() - items, err := upsertService(false, prod, svc, nil, renderSet, kubeClient, k.log) + items, err := upsertService(false, prod, svc, nil, renderSet, informer, kubeClient, istioClient, k.log) if err != nil { lock.Lock() switch e := err.(type) { diff --git a/pkg/microservice/aslan/core/environment/service/kube.go b/pkg/microservice/aslan/core/environment/service/kube.go index 7c8e0fa399b5ec7db6b531ee6da202e7127b5ab2..7030a1a3137708502c3ed8492dd33db29bc767d0 100644 --- a/pkg/microservice/aslan/core/environment/service/kube.go +++ b/pkg/microservice/aslan/core/environment/service/kube.go @@ -21,14 +21,18 @@ import ( "time" "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/sets" + "github.com/koderover/zadig/pkg/microservice/aslan/config" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" "github.com/koderover/zadig/pkg/shared/kube/resource" "github.com/koderover/zadig/pkg/shared/kube/wrapper" e "github.com/koderover/zadig/pkg/tool/errors" @@ -107,9 +111,9 @@ func ListPodEvents(envName, productName, podName string, log *zap.SugaredLogger) } // ListAvailableNamespaces lists available namespaces created by non-koderover -func ListAvailableNamespaces(clusterID string, log *zap.SugaredLogger) ([]*resource.Namespace, error) { +func ListAvailableNamespaces(clusterID, listType string, log *zap.SugaredLogger) ([]*resource.Namespace, error) { resp := make([]*resource.Namespace, 0) - kubeClient, err := kube.GetKubeClient(clusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), clusterID) if err != nil { log.Errorf("ListNamespaces clusterID:%s err:%v", clusterID, err) return resp, err @@ -118,11 +122,19 @@ func ListAvailableNamespaces(clusterID string, log *zap.SugaredLogger) ([]*resou if err != nil { log.Errorf("ListNamespaces err:%v", err) if apierrors.IsForbidden(err) { - return resp, nil + return resp, err } return resp, err } - + filterK8sNamespaces := sets.NewString("kube-node-lease", "kube-public", "kube-system") + if listType == setting.ListNamespaceTypeCreate { + nsList, err := commonrepo.NewProductColl().ListExistedNamespace() + if err != nil { + log.Errorf("Failed to list existed namespace from the env List, error: %s", err) + return nil, err + } + filterK8sNamespaces.Insert(nsList...) + } for _, namespace := range namespaces { if value, IsExist := namespace.Labels[setting.EnvCreatedBy]; IsExist { if value == setting.EnvCreator { @@ -130,6 +142,10 @@ func ListAvailableNamespaces(clusterID string, log *zap.SugaredLogger) ([]*resou } } + if filterK8sNamespaces.Has(namespace.Name) { + continue + } + resp = append(resp, wrapper.Namespace(namespace).Resource()) } @@ -146,7 +162,7 @@ func ListServicePods(productName, envName string, serviceName string, log *zap.S if err != nil { return res, e.ErrListServicePod.AddErr(err) } - kubeClient, err := kube.GetKubeClient(product.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), product.ClusterID) if err != nil { return res, e.ErrListServicePod.AddErr(err) } @@ -173,7 +189,7 @@ func DeletePod(envName, productName, podName string, log *zap.SugaredLogger) err if err != nil { return e.ErrDeletePod.AddErr(err) } - kubeClient, err := kube.GetKubeClient(product.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), product.ClusterID) if err != nil { return e.ErrDeletePod.AddErr(err) } @@ -223,3 +239,56 @@ func getModifiedServiceFromObjectMeta(om metav1.Object) *serviceInfo { LastUpdateTime: t, } } + +func ListAvailableNodes(clusterID string, log *zap.SugaredLogger) (*NodeResp, error) { + resp := new(NodeResp) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), clusterID) + if err != nil { + log.Errorf("ListAvailableNodes clusterID:%s err:%s", clusterID, err) + return resp, err + } + + nodes, err := getter.ListNodes(kubeClient) + if err != nil { + log.Errorf("ListNodes err:%s", err) + if apierrors.IsForbidden(err) { + return resp, err + } + return resp, err + } + + nodeInfos := make([]*resource.Node, 0) + labels := sets.NewString() + for _, node := range nodes { + nodeResource := &resource.Node{ + Ready: nodeReady(node), + Labels: nodeLabel(node), + IP: node.Name, + } + nodeInfos = append(nodeInfos, nodeResource) + labels.Insert(nodeResource.Labels...) + } + resp.Nodes = nodeInfos + resp.Labels = labels.List() + return resp, nil +} + +// Ready indicates that the node is ready for traffic. +func nodeReady(node *corev1.Node) bool { + cs := node.Status.Conditions + for _, c := range cs { + if c.Type == corev1.NodeReady && c.Status == corev1.ConditionTrue { + return true + } + } + return false +} + +func nodeLabel(node *corev1.Node) []string { + labels := make([]string, 0, len(node.Labels)) + labelM := node.Labels + for key, value := range labelM { + labels = append(labels, fmt.Sprintf("%s:%s", key, value)) + } + return labels +} diff --git a/pkg/microservice/aslan/core/environment/service/pm.go b/pkg/microservice/aslan/core/environment/service/pm.go index 27c810e370b83924a6f4bcfd15159b7402d483ed..253f4457f62e87b5453a893b815161937a273d98 100644 --- a/pkg/microservice/aslan/core/environment/service/pm.go +++ b/pkg/microservice/aslan/core/environment/service/pm.go @@ -24,12 +24,16 @@ import ( "github.com/hashicorp/go-multierror" "go.uber.org/zap" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/informers" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/pm" "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" "github.com/koderover/zadig/pkg/setting" e "github.com/koderover/zadig/pkg/tool/errors" @@ -40,7 +44,7 @@ type PMService struct { log *zap.SugaredLogger } -func (p *PMService) queryServiceStatus(namespace, envName, productName string, serviceTmpl *commonmodels.Service, kubeClient client.Client) (string, string, []string) { +func (p *PMService) queryServiceStatus(namespace, envName, productName string, serviceTmpl *commonmodels.Service, informer informers.SharedInformerFactory) (string, string, []string) { p.log.Infof("queryServiceStatus of service: %s of product: %s in namespace %s", serviceTmpl.ServiceName, productName, namespace) pipelineName := fmt.Sprintf("%s-%s-%s", serviceTmpl.ServiceName, envName, "job") taskObj, err := commonrepo.NewTaskColl().FindTask(pipelineName, config.ServiceType) @@ -51,7 +55,7 @@ func (p *PMService) queryServiceStatus(namespace, envName, productName string, s return setting.PodPending, setting.PodNotReady, []string{} } - return queryPodsStatus(namespace, "", serviceTmpl.ServiceName, kubeClient, p.log) + return queryPodsStatus(namespace, "", serviceTmpl.ServiceName, informer, p.log) } func (p *PMService) updateService(args *SvcOptArgs) error { @@ -85,7 +89,7 @@ func (p *PMService) updateService(args *SvcOptArgs) error { return nil } -func (p *PMService) listGroupServices(allServices []*commonmodels.ProductService, envName, productName string, kubeClient client.Client, productInfo *commonmodels.Product) []*commonservice.ServiceResp { +func (p *PMService) listGroupServices(allServices []*commonmodels.ProductService, envName, productName string, informer informers.SharedInformerFactory, productInfo *commonmodels.Product) []*commonservice.ServiceResp { var wg sync.WaitGroup var resp []*commonservice.ServiceResp var mutex sync.RWMutex @@ -98,10 +102,12 @@ func (p *PMService) listGroupServices(allServices []*commonmodels.ProductService ServiceName: service.ServiceName, Type: service.Type, EnvName: envName, + Revision: service.Revision, } serviceTmpl, err := commonservice.GetServiceTemplate( service.ServiceName, setting.PMDeployType, service.ProductName, "", service.Revision, p.log, ) + if err != nil { gp.Status = setting.PodFailed mutex.Lock() @@ -110,14 +116,30 @@ func (p *PMService) listGroupServices(allServices []*commonmodels.ProductService return } + for _, envconfig := range serviceTmpl.EnvConfigs { + if envconfig.EnvName == envName { + gp.EnvConfigs = []*models.EnvConfig{envconfig} + break + } + } + gp.ProductName = serviceTmpl.ProductName if len(serviceTmpl.EnvStatuses) > 0 { envStatuses := make([]*commonmodels.EnvStatus, 0) + filterEnvStatuses, err := pm.GenerateEnvStatus(serviceTmpl.EnvConfigs, log.NopSugaredLogger()) + if err != nil { + return + } + filterEnvStatusSet := sets.NewString() + for _, v := range filterEnvStatuses { + filterEnvStatusSet.Insert(v.Address) + } for _, envStatus := range serviceTmpl.EnvStatuses { - if envStatus.EnvName == envName { + if envStatus.EnvName == envName && filterEnvStatusSet.Has(envStatus.Address) { envStatuses = append(envStatuses, envStatus) } } + if len(envStatuses) > 0 { gp.EnvStatuses = envStatuses mutex.Lock() @@ -141,7 +163,7 @@ func (p *PMService) listGroupServices(allServices []*commonmodels.ProductService return resp } -func (p *PMService) createGroup(envName, productName, username string, group []*commonmodels.ProductService, renderSet *commonmodels.RenderSet, kubeClient client.Client) error { +func (p *PMService) createGroup(envName, productName, username string, group []*commonmodels.ProductService, renderSet *commonmodels.RenderSet, inf informers.SharedInformerFactory, kubeClient client.Client) error { p.log.Infof("[Namespace:%s][Product:%s] createGroup", envName, productName) // 异步创建无依赖的服务 @@ -161,14 +183,32 @@ func (p *PMService) createGroup(envName, productName, username string, group []* } if serviceTempl != nil { oldEnvConfigs := serviceTempl.EnvConfigs + newEnvConfigs := []*commonmodels.EnvConfig{} + // rm not exist env + for _, v := range oldEnvConfigs { + if _, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: productName, + EnvName: v.EnvName, + }); err == nil { + if envName != v.EnvName { + newEnvConfigs = append(newEnvConfigs, v) + } + } + } for _, currentEnvConfig := range productService.EnvConfigs { envConfig := &commonmodels.EnvConfig{ EnvName: currentEnvConfig.EnvName, HostIDs: currentEnvConfig.HostIDs, + Labels: currentEnvConfig.Labels, } - oldEnvConfigs = append(oldEnvConfigs, envConfig) + newEnvConfigs = append(newEnvConfigs, envConfig) } + changeEnvStatus, err := pm.GenerateEnvStatus(newEnvConfigs, log.NopSugaredLogger()) + if err != nil { + log.Errorf("GenerateEnvStatus err:%s", err) + return err + } args := &commonservice.ServiceTmplBuildObject{ ServiceTmplObject: &commonservice.ServiceTmplObject{ ProductName: serviceTempl.ProductName, @@ -178,8 +218,8 @@ func (p *PMService) createGroup(envName, productName, username string, group []* Type: serviceTempl.Type, Username: username, HealthChecks: serviceTempl.HealthChecks, - EnvConfigs: oldEnvConfigs, - EnvStatuses: []*commonmodels.EnvStatus{}, + EnvConfigs: newEnvConfigs, + EnvStatuses: changeEnvStatus, From: "createEnv", }, Build: &commonmodels.Build{Name: serviceTempl.BuildName}, @@ -224,10 +264,5 @@ func (p *PMService) createGroup(envName, productName, username string, group []* errList = multierror.Append(errList, err) } } - // 如果创建依赖服务组有返回错误, 停止等待 - if err := errList.ErrorOrNil(); err != nil { - return err - } - - return nil + return errList.ErrorOrNil() } diff --git a/pkg/microservice/aslan/core/environment/service/pm_exec.go b/pkg/microservice/aslan/core/environment/service/pm_exec.go new file mode 100644 index 0000000000000000000000000000000000000000..bff1b90846e0e1aed2894c00efc0cc9832b52765 --- /dev/null +++ b/pkg/microservice/aslan/core/environment/service/pm_exec.go @@ -0,0 +1,91 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "encoding/base64" + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/gorilla/websocket" + "go.uber.org/zap" + + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/setting" + e "github.com/koderover/zadig/pkg/tool/errors" + toolssh "github.com/koderover/zadig/pkg/tool/ssh" + "github.com/koderover/zadig/pkg/tool/wsconn" +) + +var upgrader = websocket.Upgrader{ + ReadBufferSize: 1024, + WriteBufferSize: 1024, + CheckOrigin: func(r *http.Request) bool { + return true + }, +} + +func ConnectSshPmExec(c *gin.Context, username, envName, productName, ip, hostId string, cols, rows int, log *zap.SugaredLogger) error { + resp, err := commonrepo.NewPrivateKeyColl().Find(commonrepo.FindPrivateKeyOption{ + ID: hostId, + }) + if err != nil { + log.Errorf("PrivateKey.Find ip %s id %s error: %s", ip, hostId, err) + return e.ErrGetPrivateKey + } + if resp.Status != setting.PMHostStatusNormal { + return e.ErrLoginPm.AddDesc(fmt.Sprintf("host %s status %s,is not normal", ip, resp.Status)) + } + if resp.Port == 0 { + resp.Port = setting.PMHostDefaultPort + } + + sDec, err := base64.StdEncoding.DecodeString(resp.PrivateKey) + if err != nil { + log.Errorf("base64 decode failed ip:%s, error:%s", ip, err) + return e.ErrLoginPm.AddDesc(fmt.Sprintf("base64 decode failed ip:%s, error:%s", ip, err)) + } + + sshCli, err := toolssh.NewSshCli(sDec, resp.UserName, resp.IP, resp.Port) + if err != nil { + log.Errorf("NewSshCli err:%s", err) + return e.ErrLoginPm.AddErr(err) + } + + sshConn, err := wsconn.NewSshConn(cols, rows, sshCli) + if err != nil { + log.Errorf("NewSshConn err:%s", err) + return e.ErrLoginPm.AddErr(err) + } + defer sshConn.Close() + + ws, err := upgrader.Upgrade(c.Writer, c.Request, nil) + if err != nil { + log.Errorf("ws upgrade err:%s", err) + return e.ErrLoginPm.AddErr(err) + } + defer ws.Close() + + stopChan := make(chan bool, 3) + go sshConn.ReadWsMessage(ws, stopChan) + go sshConn.SendWsWriteMessage(ws, stopChan) + go sshConn.SessionWait(stopChan) + + <-stopChan + return nil +} diff --git a/pkg/microservice/aslan/core/environment/service/product.go b/pkg/microservice/aslan/core/environment/service/product.go index 52909896ee3cd78bfc85d88daf7c265ef493d9b1..27d680ae219a55fd7d1ac1062dca4a5f009c1eb6 100644 --- a/pkg/microservice/aslan/core/environment/service/product.go +++ b/pkg/microservice/aslan/core/environment/service/product.go @@ -17,6 +17,7 @@ limitations under the License. package service import ( + "context" "fmt" "time" @@ -29,11 +30,13 @@ import ( commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/collaboration" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/poetry" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types" + "github.com/koderover/zadig/pkg/util" ) var DefaultCleanWhiteList = []string{"spockadmin"} @@ -48,7 +51,10 @@ func CleanProductCronJob(requestID string, log *zap.SugaredLogger) { log.Errorf("[Product.List] error: %v", err) return } - + envCMMap, err := collaboration.GetEnvCMMap([]string{}, log) + if err != nil { + return + } wl := sets.NewString(DefaultCleanWhiteList...) wl.Insert(config.CleanSkippedList()...) for _, product := range products { @@ -59,33 +65,29 @@ func CleanProductCronJob(requestID string, log *zap.SugaredLogger) { if product.RecycleDay == 0 { continue } - + if _, ok := envCMMap[collaboration.BuildEnvCMMapKey(product.ProductName, product.EnvName)]; ok { + continue + } if time.Now().Unix()-product.UpdateTime > int64(60*60*24*product.RecycleDay) { - title := "系统清理产品信息" - content := fmt.Sprintf("环境 [%s] 已经连续%d天没有使用, 系统已自动删除该环境, 如有需要请重新创建。", product.EnvName, product.RecycleDay) + //title := "系统清理产品信息" + //content := fmt.Sprintf("环境 [%s] 已经连续%d天没有使用, 系统已自动删除该环境, 如有需要请重新创建。", product.EnvName, product.RecycleDay) - if err := commonservice.DeleteProduct("robot", product.EnvName, product.ProductName, requestID, log); err != nil { + if err := DeleteProduct("robot", product.EnvName, product.ProductName, requestID, true, log); err != nil { log.Errorf("[%s][P:%s] delete product error: %v", product.EnvName, product.ProductName, err) // 如果有错误，重试删除 - if err := commonservice.DeleteProduct("robot", product.EnvName, product.ProductName, requestID, log); err != nil { - content = fmt.Sprintf("系统自动清理环境 [%s] 失败，请手动删除环境。", product.ProductName) + if err := DeleteProduct("robot", product.EnvName, product.ProductName, requestID, true, log); err != nil { + //content = fmt.Sprintf("系统自动清理环境 [%s] 失败，请手动删除环境。", product.ProductName) log.Errorf("[%s][P:%s] retry delete product error: %v", product.EnvName, product.ProductName, err) } } - poetryClient := poetry.New(config.PoetryAPIServer(), config.PoetryAPIRootKey()) - users, _ := poetryClient.ListProductPermissionUsers("", "", log) - for _, user := range users { - commonservice.SendMessage(user, title, content, requestID, log) - } - log.Warnf("[%s] product %s deleted", product.EnvName, product.ProductName) } } } -func GetInitProduct(productTmplName string, log *zap.SugaredLogger) (*commonmodels.Product, error) { +func GetInitProduct(productTmplName string, envType types.EnvType, isBaseEnv bool, baseEnvName string, log *zap.SugaredLogger) (*commonmodels.Product, error) { ret := &commonmodels.Product{} prodTmpl, err := templaterepo.NewProductColl().Find(productTmplName) @@ -108,11 +110,9 @@ func GetInitProduct(productTmplName string, log *zap.SugaredLogger) (*commonmode //返回中的ProductName即产品模板的名称 ret.ProductName = prodTmpl.ProductName ret.Revision = prodTmpl.Revision - ret.Enabled = prodTmpl.Enabled ret.Services = [][]*commonmodels.ProductService{} ret.UpdateBy = prodTmpl.UpdateBy ret.CreateTime = prodTmpl.CreateTime - ret.Visibility = prodTmpl.Visibility ret.Render = &commonmodels.RenderInfo{Name: "", Description: ""} ret.Vars = prodTmpl.Vars ret.ChartInfos = prodTmpl.ChartInfos @@ -120,8 +120,37 @@ func GetInitProduct(productTmplName string, log *zap.SugaredLogger) (*commonmode ret.Source = setting.PMDeployType } + svcGroupNames := prodTmpl.Services + if envType == types.ShareEnv && !isBaseEnv { + // At this point the request is from the environment share. + svcGroupNames, err = GetEnvServiceList(context.TODO(), productTmplName, baseEnvName) + if err != nil { + return nil, fmt.Errorf("failed to get service list from env %s of product %s: %s", baseEnvName, productTmplName, err) + } + + // Note: In the Helm scenario, filter `chart_infos` which is used by front-end. + if prodTmpl.ProductFeature != nil && prodTmpl.ProductFeature.DeployType == setting.HelmDeployType { + chartInfos := []*templatemodels.RenderChart{} + for _, chartInfo := range ret.ChartInfos { + found := false + for _, svcGroupName := range svcGroupNames { + if util.InStringArray(chartInfo.ServiceName, svcGroupName) { + found = true + break + } + } + + if found { + chartInfos = append(chartInfos, chartInfo) + } + } + + ret.ChartInfos = chartInfos + } + } + allServiceInfoMap := prodTmpl.AllServiceInfoMap() - for _, names := range prodTmpl.Services { + for _, names := range svcGroupNames { servicesResp := make([]*commonmodels.ProductService, 0) for _, serviceName := range names { @@ -151,6 +180,7 @@ func GetInitProduct(productTmplName string, log *zap.SugaredLogger) (*commonmode Name: c.Name, Image: c.Image, ImagePath: c.ImagePath, + ImageName: util.GetImageNameFromContainerInfo(c.ImageName, c.Name), } serviceResp.Containers = append(serviceResp.Containers, container) } @@ -167,7 +197,7 @@ func GetProduct(username, envName, productName string, log *zap.SugaredLogger) ( opt := &commonrepo.ProductFindOptions{Name: productName, EnvName: envName} prod, err := commonrepo.NewProductColl().Find(opt) if err != nil { - log.Errorf("[User:%s][EnvName:%s][Product:%s] Product.FindByOwner error: %v", username, envName, productName, err) + log.Errorf("[User:%s][EnvName:%s][Product:%s] Product.FindByOwner error: %s", username, envName, productName, err) return nil, e.ErrGetEnv } @@ -178,27 +208,40 @@ func GetProduct(username, envName, productName string, log *zap.SugaredLogger) ( } } + if len(prod.RegistryID) == 0 { + reg, _, err := commonservice.FindDefaultRegistry(false, log) + if err != nil { + log.Errorf("[User:%s][EnvName:%s][Product:%s] FindDefaultRegistry error: %s", username, envName, productName, err) + return nil, err + } + prod.RegistryID = reg.ID.Hex() + } resp := buildProductResp(prod.EnvName, prod, log) return resp, nil } func buildProductResp(envName string, prod *commonmodels.Product, log *zap.SugaredLogger) *ProductResp { prodResp := &ProductResp{ - ID: prod.ID.Hex(), - ProductName: prod.ProductName, - Namespace: prod.Namespace, - Services: [][]string{}, - Status: setting.PodUnstable, - EnvName: prod.EnvName, - UpdateTime: prod.UpdateTime, - UpdateBy: prod.UpdateBy, - Render: prod.Render, - Error: prod.Error, - Vars: prod.Vars[:], - IsPublic: prod.IsPublic, - ClusterID: prod.ClusterID, - RecycleDay: prod.RecycleDay, - Source: prod.Source, + ID: prod.ID.Hex(), + ProductName: prod.ProductName, + Namespace: prod.Namespace, + Services: [][]string{}, + Status: setting.PodUnstable, + EnvName: prod.EnvName, + UpdateTime: prod.UpdateTime, + UpdateBy: prod.UpdateBy, + Render: prod.Render, + Error: prod.Error, + Vars: prod.Vars[:], + IsPublic: prod.IsPublic, + IsExisted: prod.IsExisted, + ClusterID: prod.ClusterID, + RecycleDay: prod.RecycleDay, + Source: prod.Source, + RegisterID: prod.RegistryID, + ShareEnvEnable: prod.ShareEnv.Enable, + ShareEnvIsBase: prod.ShareEnv.IsBase, + ShareEnvBaseEnv: prod.ShareEnv.BaseEnv, } if prod.ClusterID != "" { @@ -215,12 +258,20 @@ func buildProductResp(envName string, prod *commonmodels.Product, log *zap.Sugar return prodResp } prodResp.IsProd = cluster.Production + prodResp.ClusterName = cluster.Name + prodResp.IsLocal = cluster.Local - if !clusterService.ClusterConnected(prod.ClusterID) { + if !prodResp.IsLocal && !clusterService.ClusterConnected(prod.ClusterID) { prodResp.Status = setting.ClusterDisconnected prodResp.Error = "集群未连接" return prodResp } + } else { + prodResp.IsLocal = true + } + + if prod.Source != setting.SourceFromExternal { + prodResp.Services = prod.GetGroupServiceNames() } if prod.Status == setting.ProductStatusCreating { @@ -254,7 +305,6 @@ func buildProductResp(envName string, prod *commonmodels.Product, log *zap.Sugar return prodResp } default: - prodResp.Services = prod.GetGroupServiceNames() servicesResp, _, errObj = ListGroups("", envName, prod.ProductName, -1, -1, log) } @@ -292,8 +342,8 @@ func CleanProducts() { for _, prod := range products { _, err := templaterepo.NewProductColl().Find(prod.ProductName) if err != nil && err.Error() == "not found" { - logger.Errorf("集成环境所属的项目不存在，准备删除此集成环境, namespace:%s, 项目:%s\n", prod.Namespace, prod.ProductName) - err = commonservice.DeleteProduct("CleanProducts", prod.EnvName, prod.ProductName, "", logger) + logger.Errorf("环境所属的项目不存在，准备删除此环境, namespace:%s, 项目:%s\n", prod.Namespace, prod.ProductName) + err = DeleteProduct("CleanProducts", prod.EnvName, prod.ProductName, "", true, logger) if err != nil { logger.Errorf("delete product failed, namespace:%s, err:%v\n", prod.Namespace, err) continue diff --git a/pkg/microservice/aslan/core/environment/service/pvc.go b/pkg/microservice/aslan/core/environment/service/pvc.go new file mode 100644 index 0000000000000000000000000000000000000000..8b910952bf83a6c569d529eb09877cb09ede039f --- /dev/null +++ b/pkg/microservice/aslan/core/environment/service/pvc.go @@ -0,0 +1,225 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "encoding/json" + "sort" + "sync" + + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" + "sigs.k8s.io/yaml" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/kube/getter" + "github.com/koderover/zadig/pkg/tool/kube/updater" +) + +type ListPvcsResponse struct { + *ResourceResponseBase + PvcName string `json:"pvc_name"` + Status string `json:"status"` + Volume string `json:"volume"` + AccessModes string `json:"access_modes"` + StorageClass string `json:"storageclass"` + Capacity string `json:"capacity"` +} + +func ListPvcs(envName, productName string, log *zap.SugaredLogger) ([]*ListPvcsResponse, error) { + product, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: productName, + EnvName: envName, + }) + if err != nil { + return nil, e.ErrListResources.AddErr(err) + } + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), product.ClusterID) + if err != nil { + return nil, e.ErrListResources.AddErr(err) + } + + Pvcs, err := getter.ListPvcs(product.Namespace, nil, kubeClient) + if err != nil { + log.Error(err) + return nil, e.ErrListResources.AddDesc(err.Error()) + } + envSvcDepends, err := commonrepo.NewEnvSvcDependColl().List(&commonrepo.ListEnvSvcDependOption{ProductName: productName, EnvName: envName}) + if err != nil && commonrepo.IsErrNoDocuments(err) { + return nil, e.ErrListResources.AddDesc(err.Error()) + } + + var res []*ListPvcsResponse + var mutex sync.Mutex + var wg sync.WaitGroup + pvcProcess := func(pvc *v1.PersistentVolumeClaim) error { + pvc.SetManagedFields(nil) + pvc.SetResourceVersion("") + if pvc.APIVersion == "" { + pvc.APIVersion = "v1" + } + if pvc.Kind == "" { + pvc.Kind = "PersistentVolumeClaim" + } + yamlData, err := yaml.Marshal(pvc) + if err != nil { + return err + } + accessModes := "" + lenAm := len(pvc.Spec.AccessModes) + for i, am := range pvc.Spec.AccessModes { + accessModes += string(am) + if i < lenAm-1 { + accessModes += "," + } + } + var tempSvcs []string + LB: + for _, svcDepend := range envSvcDepends { + for _, pvcc := range svcDepend.Pvcs { + if pvcc == pvc.Name { + tempSvcs = append(tempSvcs, svcDepend.ServiceModule) + continue LB + } + } + } + + storageClassName := "" + if pvc.Spec.StorageClassName != nil { + storageClassName = *pvc.Spec.StorageClassName + } + capacity := "" + resourceList := pvc.Status.Capacity[v1.ResourceStorage] + if &resourceList != nil { + capacity = resourceList.String() + } + resElem := &ListPvcsResponse{ + PvcName: pvc.Name, + Status: string(pvc.Status.Phase), + Volume: pvc.Spec.VolumeName, + AccessModes: accessModes, + StorageClass: storageClassName, + Capacity: capacity, + ResourceResponseBase: &ResourceResponseBase{ + Name: pvc.Name, + Type: config.CommonEnvCfgTypePvc, + EnvName: envName, + ProjectName: productName, + YamlData: string(yamlData), + Services: tempSvcs, + CreateTime: pvc.GetCreationTimestamp().Time, + }, + } + resElem.setSourceDetailData(pvc) + mutex.Lock() + res = append(res, resElem) + mutex.Unlock() + return nil + } + for _, pvcElem := range Pvcs { + wg.Add(1) + go func(pvc *v1.PersistentVolumeClaim) { + defer wg.Done() + err := pvcProcess(pvc) + if err != nil { + log.Errorf("ListPvcs ns:%s name:%s pvcProcess err:%s", pvc.Namespace, pvc.Name, err) + } + }(pvcElem) + } + wg.Wait() + sort.SliceStable(res, func(i, j int) bool { + return res[i].PvcName < res[j].PvcName + }) + return res, nil +} + +type UpdatePvcArgs struct { + EnvName string `json:"env_name"` + ProductName string `json:"product_name"` + PvcName string `json:"pvc_name"` + YamlData string `json:"yaml_data"` + RestartAssociatedSvc bool `json:"restart_associated_svc"` +} + +func UpdatePvc(args *models.CreateUpdateCommonEnvCfgArgs, userName string, log *zap.SugaredLogger) error { + js, err := yaml.YAMLToJSON([]byte(args.YamlData)) + pvc := &corev1.PersistentVolumeClaim{} + err = json.Unmarshal(js, pvc) + if err != nil { + return e.ErrUpdateResource.AddErr(err) + } + if pvc.Name != args.Name { + return e.ErrUpdateResource.AddDesc("pvc Yaml Name is incorrect") + } + + product, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: args.ProductName, + EnvName: args.EnvName, + }) + if err != nil { + return e.ErrUpdateResource.AddErr(err) + } + + clientset, err := kubeclient.GetKubeClientSet(config.HubServerAddress(), product.ClusterID) + if err != nil { + log.Errorf("failed to create kubernetes clientset for clusterID: %s, the error is: %s", product.ClusterID, err) + return e.ErrUpdateResource.AddErr(err) + } + + yamlData, err := ensureLabelAndNs(pvc, product.Namespace, args.ProductName) + if err != nil { + return e.ErrUpdateResource.AddErr(err) + } + + err = updater.UpdatePvc(product.Namespace, pvc, clientset) + if err != nil { + log.Error(err) + return e.ErrUpdateResource.AddDesc(err.Error()) + } + envPvc := &models.EnvResource{ + ProductName: args.ProductName, + UpdateUserName: userName, + EnvName: args.EnvName, + Namespace: product.Namespace, + Name: pvc.Name, + YamlData: yamlData, + Type: string(config.CommonEnvCfgTypePvc), + SourceDetail: args.SourceDetail, + AutoSync: args.AutoSync, + } + if commonrepo.NewEnvResourceColl().Create(envPvc) != nil { + return e.ErrUpdateResource.AddDesc(err.Error()) + } + + if !args.RestartAssociatedSvc { + return nil + } + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), product.ClusterID) + if err != nil { + return e.ErrUpdateResource.AddErr(err) + } + + if err := restartPod(pvc.Name, args.ProductName, args.EnvName, product.Namespace, config.CommonEnvCfgTypeSecret, clientset, kubeClient); err != nil { + return e.ErrRestartService.AddDesc(err.Error()) + } + return nil +} diff --git a/pkg/microservice/aslan/core/environment/service/renderset.go b/pkg/microservice/aslan/core/environment/service/renderset.go index 5bd11d00736d7e8f2c9444967e81aaa35548093c..d97ca9487b72b3c06a51df66d52c7396292e3d86 100644 --- a/pkg/microservice/aslan/core/environment/service/renderset.go +++ b/pkg/microservice/aslan/core/environment/service/renderset.go @@ -18,217 +18,168 @@ package service import ( "fmt" - "strings" "sync" "github.com/pkg/errors" + "go.mongodb.org/mongo-driver/mongo" "go.uber.org/zap" - "gopkg.in/yaml.v3" - "k8s.io/apimachinery/pkg/util/sets" - commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" + templatemodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" - commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" fsservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/fs" "github.com/koderover/zadig/pkg/setting" - e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/log" yamlutil "github.com/koderover/zadig/pkg/util/yaml" ) -func GetRenderCharts(productName, envName, serviceName string, log *zap.SugaredLogger) ([]*commonservice.RenderChartArg, error) { +type DefaultValuesResp struct { + DefaultValues string `json:"defaultValues"` + YamlData *templatemodels.CustomYaml `json:"yaml_data,omitempty"` +} - renderSetName := commonservice.GetProductEnvNamespace(envName, productName, "") +type YamlContentRequestArg struct { + CodehostID int `json:"codehostID" form:"codehostID"` + Owner string `json:"owner" form:"owner"` + Repo string `json:"repo" form:"repo"` + Namespace string `json:"namespace" form:"namespace"` + Branch string `json:"branch" form:"branch"` + RepoLink string `json:"repoLink" form:"repoLink"` + ValuesPaths string `json:"valuesPaths" form:"valuesPaths"` +} - opt := &commonrepo.RenderSetFindOption{ - Name: renderSetName, +func fromGitRepo(source string) bool { + if source == "" { + return true } - rendersetObj, existed, err := commonrepo.NewRenderSetColl().FindRenderSet(opt) - if err != nil { - return nil, err + if source == setting.SourceFromGitRepo { + return true } + return false +} - if !existed { - return nil, nil +// SyncYamlFromSource sync values.yaml from source +// NOTE currently only support gitHub and gitlab +func SyncYamlFromSource(yamlData *templatemodels.CustomYaml, curValue string) (bool, string, error) { + if yamlData == nil || !yamlData.AutoSync { + return false, "", nil } - - ret := make([]*commonservice.RenderChartArg, 0) - - matchedRenderChartModels := make([]*template.RenderChart, 0) - if len(serviceName) == 0 { - matchedRenderChartModels = rendersetObj.ChartInfos - } else { - serverList := strings.Split(serviceName, ",") - stringSet := sets.NewString(serverList...) - for _, singleChart := range rendersetObj.ChartInfos { - if !stringSet.Has(singleChart.ServiceName) { - continue - } - matchedRenderChartModels = append(matchedRenderChartModels, singleChart) - } + if !fromGitRepo(yamlData.Source) { + return false, "", nil } - for _, singleChart := range matchedRenderChartModels { - rcaObj := new(commonservice.RenderChartArg) - rcaObj.LoadFromRenderChartModel(singleChart) - rcaObj.EnvName = envName - ret = append(ret, rcaObj) + sourceDetail, err := service.UnMarshalSourceDetail(yamlData.SourceDetail) + if err != nil { + return false, "", err + } + if sourceDetail.GitRepoConfig == nil { + log.Warnf("git repo config is nil") + return false, "", nil + } + repoConfig := sourceDetail.GitRepoConfig + + valuesYAML, err := fsservice.DownloadFileFromSource(&fsservice.DownloadFromSourceArgs{ + CodehostID: repoConfig.CodehostID, + Namespace: repoConfig.Namespace, + Owner: repoConfig.Owner, + Repo: repoConfig.Repo, + Path: sourceDetail.LoadPath, + Branch: repoConfig.Branch, + }) + if err != nil { + return false, "", err } - return ret, nil -} - -// validate yaml content -func validateYamlContent(yamlContent string) error { - tMap := map[string]interface{}{} - if err := yaml.Unmarshal([]byte(yamlContent), &tMap); err != nil { - if err != nil { - return errors.New("yaml content illegal") - } + equal, err := yamlutil.Equal(string(valuesYAML), curValue) + if err != nil || equal { + return false, "", err } - return nil -} - -func generateValuesYaml(args *commonservice.RenderChartArg, log *zap.SugaredLogger) (string, error) { - if args.YamlSource == setting.ValuesYamlSourceFreeEdit { - return args.ValuesYAML, validateYamlContent(args.ValuesYAML) - } else if args.YamlSource == setting.ValuesYamlSourceGitRepo { - if args.GitRepoConfig == nil { - return "", errors.New("invalid repo config") - } - var ( - allValues []byte - fileContentMap sync.Map - wg sync.WaitGroup - err error - ) - - for index, filePath := range args.GitRepoConfig.ValuesPaths { - wg.Add(1) - go func(index int, path string) { - defer wg.Done() - fileContent, err1 := fsservice.DownloadFileFromSource( - &fsservice.DownloadFromSourceArgs{ - CodehostID: args.GitRepoConfig.CodehostID, - Owner: args.GitRepoConfig.Owner, - Repo: args.GitRepoConfig.Repo, - Path: path, - Branch: args.GitRepoConfig.Branch, - }) - if err1 != nil { - err = errors.Errorf("fail to download file from git, err: %s, path: %s, repo: %v", err1.Error(), path, *args.GitRepoConfig) - return - } - fileContentMap.Store(index, fileContent) - }(index, filePath) - } - wg.Wait() - - if err != nil { - return "", err - } - - allValueYamls := make([][]byte, len(args.GitRepoConfig.ValuesPaths), len(args.GitRepoConfig.ValuesPaths)) - for i := 0; i < len(args.GitRepoConfig.ValuesPaths); i++ { - contentObj, _ := fileContentMap.Load(i) - allValueYamls[i] = contentObj.([]byte) - } - allValues, err = yamlutil.Merge(allValueYamls) - if err != nil { - return "", errors.Errorf("failed to merge yaml files, repo: %v", *args.GitRepoConfig) - } - return string(allValues), nil - } - return "", nil + return true, string(valuesYAML), nil } -func CreateOrUpdateChartValues(productName, envName string, args *commonservice.RenderChartArg, userName, requestID string, log *zap.SugaredLogger) error { - - serviceName := args.ServiceName +func GetDefaultValues(productName, envName string, log *zap.SugaredLogger) (*DefaultValuesResp, error) { + ret := &DefaultValuesResp{} - serviceOpt := &commonrepo.ServiceFindOption{ - ProductName: productName, - ServiceName: serviceName, - Type: setting.HelmDeployType, + productInfo, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: productName, + EnvName: envName, + }) + if err == mongo.ErrNoDocuments { + return ret, nil } - serviceObj, err := commonrepo.NewServiceColl().Find(serviceOpt) if err != nil { - return e.ErrCreateRenderSet.AddDesc(fmt.Sprintf("failed to get service %s", err.Error())) - } - if serviceObj == nil { - return e.ErrCreateRenderSet.AddDesc("service not found") + log.Errorf("failed to query product info, productName %s envName %s err %s", productName, envName, err) + return nil, fmt.Errorf("failed to query product info, productName %s envName %s", productName, envName) } - if serviceObj.HelmChart == nil { - return e.ErrCreateRenderSet.AddDesc("missing helm chart info") + if productInfo.Render == nil { + return nil, fmt.Errorf("invalid product, nil render data") } - yamlContent, err := generateValuesYaml(args, log) - if err != nil { - return e.ErrCreateRenderSet.AddDesc(err.Error()) + opt := &commonrepo.RenderSetFindOption{ + Name: productInfo.Render.Name, + Revision: productInfo.Render.Revision, + ProductTmpl: productName, + EnvName: productInfo.EnvName, } - args.ValuesYAML = yamlContent - - renderSetName := commonservice.GetProductEnvNamespace(envName, productName, "") - - opt := &commonrepo.RenderSetFindOption{Name: renderSetName} - curRenderset, found, err := commonrepo.NewRenderSetColl().FindRenderSet(opt) + rendersetObj, existed, err := commonrepo.NewRenderSetColl().FindRenderSet(opt) if err != nil { - return e.ErrCreateRenderSet.AddDesc(fmt.Sprintf("failed to get renderset %s", err.Error())) - } - - if !found { - curRenderset = &commonmodels.RenderSet{ - Name: renderSetName, - EnvName: envName, - ProductTmpl: productName, - UpdateBy: userName, - IsDefault: false, - } - } - - //update or insert service values.yaml - var targetChartInfo *template.RenderChart - for _, singleChart := range curRenderset.ChartInfos { - if singleChart.ServiceName != serviceName { - continue - } - args.FillRenderChartModel(singleChart, singleChart.ChartVersion) - targetChartInfo = singleChart - break - } - if targetChartInfo == nil { - targetChartInfo = new(template.RenderChart) - targetChartInfo.ValuesYaml = serviceObj.HelmChart.ValuesYaml - args.FillRenderChartModel(targetChartInfo, serviceObj.HelmChart.Version) - curRenderset.ChartInfos = append(curRenderset.ChartInfos, targetChartInfo) - } - - //create new renderset with increased revision - err = commonservice.CreateHelmRenderSet( - curRenderset, - log, - ) - + log.Errorf("failed to query renderset info, name %s err %s", productInfo.Render.Name, err) + return nil, err + } + if !existed { + return ret, nil + } + ret.DefaultValues = rendersetObj.DefaultValues + err = service.FillGitNamespace(rendersetObj.YamlData) if err != nil { - return e.ErrCreateRenderSet.AddDesc(err.Error()) + // Note, since user can always reselect the git info, error should not block normal logic + log.Warnf("failed to fill git namespace data, err: %s", err) } - return err + ret.YamlData = rendersetObj.YamlData + return ret, nil } -func ensureHelmRenderSetArgs(args *commonmodels.RenderSet) error { - if args == nil { - return errors.New("nil RenderSet") +func GetMergedYamlContent(arg *YamlContentRequestArg, paths []string) (string, error) { + var ( + fileContentMap sync.Map + wg sync.WaitGroup + err error + ) + for i, filePath := range paths { + wg.Add(1) + go func(index int, path string) { + defer wg.Done() + fileContent, errDownload := fsservice.DownloadFileFromSource( + &fsservice.DownloadFromSourceArgs{ + CodehostID: arg.CodehostID, + Owner: arg.Owner, + Namespace: arg.Namespace, + Repo: arg.Repo, + Path: path, + Branch: arg.Branch, + RepoLink: arg.RepoLink, + }) + if errDownload != nil { + err = errors.Wrapf(errDownload, fmt.Sprintf("failed to download file from git, path %s", path)) + return + } + fileContentMap.Store(index, fileContent) + }(i, filePath) } + wg.Wait() - if len(args.Name) == 0 { - return errors.New("empty render set name") - } - // 设置新的版本号 - rev, err := commonrepo.NewCounterColl().GetNextSeq("renderset:" + args.Name) if err != nil { - return fmt.Errorf("get next render set revision error: %v", err) + return "", err } - args.Revision = rev - return nil + contentArr := make([][]byte, 0, len(paths)) + for i := 0; i < len(paths); i++ { + contentObj, _ := fileContentMap.Load(i) + contentArr = append(contentArr, contentObj.([]byte)) + } + ret, err := yamlutil.Merge(contentArr) + if err != nil { + return "", errors.Wrapf(err, "failed to merge files") + } + return string(ret), nil } diff --git a/pkg/microservice/aslan/core/environment/service/revision.go b/pkg/microservice/aslan/core/environment/service/revision.go index e17cc768e8c1713ab80ca13eaa64717b2c2b13f6..33fb130a83a9a09f0dfe85a35c46cd1bd81243d8 100644 --- a/pkg/microservice/aslan/core/environment/service/revision.go +++ b/pkg/microservice/aslan/core/environment/service/revision.go @@ -17,109 +17,36 @@ limitations under the License. package service import ( + "context" "fmt" "go.uber.org/zap" "k8s.io/apimachinery/pkg/util/sets" - "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/poetry" e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/util" ) -func ListProductsRevision(productName, envName string, userID int, superUser bool, log *zap.SugaredLogger) ([]*ProductRevision, error) { - var ( - err error - prodRevs = make([]*ProductRevision, 0) - products = make([]*commonmodels.Product, 0) - productNameMap map[string][]int64 - productNamespaces = sets.NewString() - ) - if superUser { - products, err = commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{ExcludeStatus: setting.ProductStatusDeleting, Name: productName, EnvName: envName}) - if err != nil { - log.Errorf("Collection.Product.List error: %v", err) - return prodRevs, e.ErrListProducts.AddDesc(err.Error()) - } - } else { - //项目下所有公开环境 - publicProducts, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{IsPublic: true, ExcludeStatus: setting.ProductStatusDeleting, Name: productName, EnvName: envName}) - if err != nil { - log.Errorf("Collection.Product.List List product error: %v", err) - return prodRevs, e.ErrListProducts.AddDesc(err.Error()) - } - for _, publicProduct := range publicProducts { - products = append(products, publicProduct) - productNamespaces.Insert(publicProduct.Namespace) - } - - poetryCtl := poetry.New(config.PoetryAPIServer(), config.PoetryAPIRootKey()) - productNameMap, err = poetryCtl.GetUserProject(userID, log) - if err != nil { - log.Errorf("Collection.Product.List GetUserProject error: %v", err) - return prodRevs, e.ErrListProducts.AddDesc(err.Error()) - } - for productName, roleIDs := range productNameMap { - //用户关联角色所关联的环境 - for _, roleID := range roleIDs { - if roleID == setting.RoleOwnerID { - tmpProducts, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{ExcludeStatus: setting.ProductStatusDeleting, Name: productName, EnvName: envName}) - if err != nil { - log.Errorf("Collection.Product.List Find product error: %v", err) - return prodRevs, e.ErrListProducts.AddDesc(err.Error()) - } - for _, product := range tmpProducts { - if !productNamespaces.Has(product.Namespace) { - products = append(products, product) - } - } - } else { - roleEnvs, err := poetryCtl.ListRoleEnvs(productName, envName, roleID, log) - if err != nil { - log.Errorf("Collection.Product.List ListRoleEnvs error: %v", err) - return prodRevs, e.ErrListProducts.AddDesc(err.Error()) - } - for _, roleEnv := range roleEnvs { - product, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{Name: productName, EnvName: roleEnv.EnvName}) - if err != nil { - log.Errorf("Collection.Product.List Find product error: %v", err) - return prodRevs, e.ErrListProducts.AddDesc(err.Error()) - } - products = append(products, product) - } - } - } - } - } - - // 获取所有服务模板最新模板信息 - allServiceTmpls, err := commonrepo.NewServiceColl().ListAllRevisions() +func ListProductsRevision(productName, envName string, log *zap.SugaredLogger) (prodRevs []*ProductRevision, err error) { + products, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{Name: productName, IsSortByProductName: true, EnvName: envName, ExcludeStatus: []string{setting.ProductStatusDeleting, setting.ProductStatusUnknown}}) if err != nil { - log.Errorf("ListAllRevisions error: %v", err) - return prodRevs, e.ErrListProducts.AddDesc(err.Error()) + return nil, e.ErrListProducts.AddDesc(err.Error()) } - // 获取所有渲染配置最新模板信息 - allRenders, err := commonrepo.NewRenderSetColl().ListAllRenders() + + // list services with max revision of project + allServiceTmpls, err := getServicesWithMaxRevision(productName) if err != nil { log.Errorf("ListAllRevisions error: %v", err) return prodRevs, e.ErrListProducts.AddDesc(err.Error()) } for _, prod := range products { - newRender := &commonmodels.RenderSet{} - if prod.Render != nil { - newRender, err = commonservice.GetRenderSet(prod.Render.Name, 0, log) - if err != nil { - return prodRevs, err - } - } - - prodRev, err := GetProductRevision(prod, allServiceTmpls, allRenders, newRender, log) + prodRev, err := GetProductRevision(prod, allServiceTmpls, log) if err != nil { log.Error(err) return prodRevs, err @@ -129,7 +56,8 @@ func ListProductsRevision(productName, envName string, userID int, superUser boo return prodRevs, nil } -func ListProductsRevisionByFacility(basicFacility string, log *zap.SugaredLogger) ([]*ProductRevision, error) { +// ListProductsRevisionByOption called by service cron +func ListProductsRevisionByOption(basicFacility string, deployType string, log *zap.SugaredLogger) ([]*ProductRevision, error) { var ( err error prodRevs = make([]*ProductRevision, 0) @@ -137,7 +65,10 @@ func ListProductsRevisionByFacility(basicFacility string, log *zap.SugaredLogger projectNames = make([]string, 0) ) - temProducts, err := templaterepo.NewProductColl().ListWithOption(&templaterepo.ProductListOpt{BasicFacility: basicFacility}) + temProducts, err := templaterepo.NewProductColl().ListWithOption(&templaterepo.ProductListOpt{ + BasicFacility: basicFacility, + DeployType: deployType, + }) if err != nil { log.Errorf("Collection.TemplateProduct.List error: %s", err) return prodRevs, e.ErrListProducts.AddDesc(err.Error()) @@ -147,36 +78,22 @@ func ListProductsRevisionByFacility(basicFacility string, log *zap.SugaredLogger projectNames = append(projectNames, v.ProductName) } - products, err = commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{ExcludeStatus: setting.ProductStatusDeleting, InProjects: projectNames}) + products, err = commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{ExcludeStatus: []string{setting.ProductStatusDeleting, setting.ProductStatusUnknown}, InProjects: projectNames}) if err != nil { log.Errorf("Collection.Product.List error: %s", err) return prodRevs, e.ErrListProducts.AddDesc(err.Error()) } - // 获取所有服务模板最新模板信息 - allServiceTmpls, err := commonrepo.NewServiceColl().ListAllRevisions() - if err != nil { - log.Errorf("ListAllRevisions error: %s", err) - return prodRevs, e.ErrListProducts.AddDesc(err.Error()) - } - - // 获取所有渲染配置最新模板信息 - allRenders, err := commonrepo.NewRenderSetColl().ListAllRenders() - if err != nil { - log.Errorf("ListAllRevisions error: %s", err) - return prodRevs, e.ErrListProducts.AddDesc(err.Error()) - } - for _, prod := range products { - newRender := &commonmodels.RenderSet{} - if prod.Render != nil { - newRender, err = commonservice.GetRenderSet(prod.Render.Name, 0, log) - if err != nil { - return prodRevs, err - } + + // find all service templates with max revisions + allServiceTmpls, err := commonrepo.NewServiceColl().ListMaxRevisions(&commonrepo.ServiceListOption{ProductName: prod.ProductName}) + if err != nil { + log.Errorf("ListAllRevisions error: %s", err) + return prodRevs, e.ErrListProducts.AddDesc(err.Error()) } - prodRev, err := GetProductRevision(prod, allServiceTmpls, allRenders, newRender, log) + prodRev, err := GetProductRevision(prod, allServiceTmpls, log) if err != nil { log.Error(err) return prodRevs, err @@ -186,13 +103,11 @@ func ListProductsRevisionByFacility(basicFacility string, log *zap.SugaredLogger return prodRevs, nil } -func GetProductRevision(product *commonmodels.Product, allServiceTmpls []*commonmodels.Service, allRender []*commonmodels.RenderSet, newRender *commonmodels.RenderSet, log *zap.SugaredLogger) (*ProductRevision, error) { - +func GetProductRevision(product *commonmodels.Product, allServiceTmpls []*commonmodels.Service, log *zap.SugaredLogger) (*ProductRevision, error) { prodRev := new(ProductRevision) - // 查询当前产品的最新模板信息 - productTemplatName := product.ProductName - prodTmpl, err := templaterepo.NewProductColl().Find(productTemplatName) + productTemplateName := product.ProductName + prodTmpl, err := templaterepo.NewProductColl().Find(productTemplateName) if err != nil { log.Errorf("[ProductTmpl.Find] %s error: %v", product.ProductName, err) return prodRev, e.ErrFindProductTmpl @@ -210,19 +125,47 @@ func GetProductRevision(product *commonmodels.Product, allServiceTmpls []*common return prodRev, nil } - // 如果当前产品版本比产品模板小, 则需要更新 if prodRev.NextRevision > prodRev.CurrentRevision { prodRev.Updatable = true } + var allRenders []*commonmodels.RenderSet + var newRender *commonmodels.RenderSet + if prodTmpl.ProductFeature == nil || prodTmpl.ProductFeature.DeployType == setting.K8SDeployType { + rendersetName := "" + if product.Render != nil { + rendersetName = product.Render.Name + newRender, err = commonservice.GetRenderSet(product.Render.Name, 0, false, product.EnvName, log) + if err != nil { + return prodRev, err + } + } + + // get all rendersets used by product services + renderRevision := sets.NewInt64() + for _, productService := range product.GetServiceMap() { + if productService.Render != nil { + renderRevision.Insert(productService.Render.Revision) + } + } + allRenders, err = commonrepo.NewRenderSetColl().ListRendersets(&commonrepo.RenderSetListOption{ + Revisions: renderRevision.List(), + ProductTmpl: productTemplateName, + RendersetName: rendersetName, + }) + if err != nil { + log.Errorf("ListAllRevisions error: %s", err) + return prodRev, e.ErrListProducts.AddDesc(err.Error()) + } + } + // 交叉对比已创建的服务组和服务组模板 - prodRev.ServiceRevisions, err = compareGroupServicesRev(prodTmpl.Services, product, allServiceTmpls, allRender, newRender, log) + prodRev.ServiceRevisions, err = compareGroupServicesRev(prodTmpl.Services, product, allServiceTmpls, allRenders, newRender, log) if err != nil { log.Error(err) return nil, e.ErrGetProductRevision.AddDesc(err.Error()) } - // 如果任一服务组有更新, 则认为产品需要更新 if !prodRev.Updatable { prodRev.Updatable = prodRev.GroupsUpdated() } @@ -241,18 +184,43 @@ func compareGroupServicesRev(servicesTmpl [][]string, productInfo *commonmodels. var serviceRev []*SvcRevision svcList := make([]*commonmodels.ProductService, 0) svcTmplNameList := make([]string, 0) - // 拍平服务组 + for _, services := range productInfo.Services { svcList = append(svcList, services...) } + for _, svcsTmpl := range servicesTmpl { svcTmplNameList = append(svcTmplNameList, svcsTmpl...) } - var err error + // Note: For sub env, only the services in the base env are displayed. + if productInfo.ShareEnv.Enable && !productInfo.ShareEnv.IsBase { + svcGroupsInBase, err := GetEnvServiceList(context.TODO(), productInfo.ProductName, productInfo.ShareEnv.BaseEnv) + if err != nil { + return nil, fmt.Errorf("failed to get service list in base env %q of product %q: %s", productInfo.EnvName, productInfo.ProductName, err) + } + + svcMap := map[string]struct{}{} + for _, svcGroup := range svcGroupsInBase { + for _, svcName := range svcGroup { + svcMap[svcName] = struct{}{} + } + } + + tmplNameList := []string{} + for _, svcTmplName := range svcTmplNameList { + if _, found := svcMap[svcTmplName]; found { + tmplNameList = append(tmplNameList, svcTmplName) + } + } + + svcTmplNameList = tmplNameList + } + + var err error serviceRev, err = compareServicesRev(svcTmplNameList, svcList, allServiceTmpls, allRender, newRender, log) if err != nil { - log.Errorf("Failed to compare service revision. Error: %v", err) + log.Errorf("Failed to compare service revision, %s:%s, Error: %v", productInfo.ProductName, productInfo.EnvName, err) return serviceRev, e.ErrListProductsRevision.AddDesc(err.Error()) } @@ -269,9 +237,9 @@ func compareServicesRev(serviceTmplNames []string, services []*commonmodels.Prod serviceRevs := make([]*SvcRevision, 0) - serviceMap := make(map[string]*commonmodels.ProductService) + productServiceMap := make(map[string]*commonmodels.ProductService) for _, service := range services { - serviceMap[service.ServiceName] = service + productServiceMap[service.ServiceName] = service } serviceTmplMap := make(map[string]string) @@ -282,7 +250,7 @@ func compareServicesRev(serviceTmplNames []string, services []*commonmodels.Prod for _, serviceTmplName := range serviceTmplNames { // 如果已创建的服务不包括新增的服务模板, 则认为是新增服务 // 新增服务如果涉及多个部署方式，目前把所有部署方式都默认加进来 - if _, ok := serviceMap[serviceTmplName]; !ok { + if _, ok := productServiceMap[serviceTmplName]; !ok { newServiceTmpls, err := getMaxServices(allServiceTmpls, serviceTmplName) if err != nil { log.Error(err) @@ -300,8 +268,9 @@ func compareServicesRev(serviceTmplNames []string, services []*commonmodels.Prod serviceRev.Containers = make([]*commonmodels.Container, 0) for _, container := range serviceTmpl.Containers { serviceRev.Containers = append(serviceRev.Containers, &commonmodels.Container{ - Image: container.Image, - Name: container.Name, + Image: container.Image, + Name: container.Name, + ImageName: util.GetImageNameFromContainerInfo(container.ImageName, container.Name), }) } } @@ -358,8 +327,9 @@ func compareServicesRev(serviceTmplNames []string, services []*commonmodels.Prod for _, container := range maxServiceTmpl.Containers { c := &commonmodels.Container{ - Image: container.Image, - Name: container.Name, + Image: container.Image, + Name: container.Name, + ImageName: util.GetImageNameFromContainerInfo(container.ImageName, container.Name), } // reuse existed container image diff --git a/pkg/microservice/aslan/core/environment/service/secret.go b/pkg/microservice/aslan/core/environment/service/secret.go new file mode 100644 index 0000000000000000000000000000000000000000..03b8efb18f72fc9a87c3310a127e340c691518b0 --- /dev/null +++ b/pkg/microservice/aslan/core/environment/service/secret.go @@ -0,0 +1,192 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "encoding/json" + "sort" + "sync" + + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/yaml" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/kube/getter" + "github.com/koderover/zadig/pkg/tool/kube/updater" +) + +type ListSecretsResponse struct { + *ResourceResponseBase + SecretName string `json:"secret_name"` + SecretType string `json:"secret_type"` +} + +func ListSecrets(envName, productName string, log *zap.SugaredLogger) ([]*ListSecretsResponse, error) { + product, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: productName, + EnvName: envName, + }) + if err != nil { + return nil, e.ErrListResources.AddErr(err) + } + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), product.ClusterID) + if err != nil { + return nil, e.ErrListResources.AddErr(err) + } + + envSvcDepends, err := commonrepo.NewEnvSvcDependColl().List(&commonrepo.ListEnvSvcDependOption{ProductName: productName, EnvName: envName}) + if err != nil && commonrepo.IsErrNoDocuments(err) { + return nil, e.ErrListResources.AddDesc(err.Error()) + } + + secrets, err := getter.ListSecrets(product.Namespace, kubeClient) + if err != nil { + log.Error(err) + return nil, e.ErrListResources.AddDesc(err.Error()) + } + + var res []*ListSecretsResponse + var mutex sync.Mutex + var wg sync.WaitGroup + secretProcess := func(secret *corev1.Secret) error { + secret.SetManagedFields(nil) + secret.SetResourceVersion("") + if secret.APIVersion == "" { + secret.APIVersion = "v1" + } + if secret.Kind == "" { + secret.Kind = "Secret" + } + yamlData, err := yaml.Marshal(secret) + if err != nil { + return err + } + var tempSvcs []string + LB: + for _, svcDepend := range envSvcDepends { + for _, secc := range svcDepend.Secrets { + if secc == secret.Name { + tempSvcs = append(tempSvcs, svcDepend.ServiceModule) + continue LB + } + } + } + resElem := &ListSecretsResponse{ + SecretName: secret.Name, + SecretType: string(secret.Type), + ResourceResponseBase: &ResourceResponseBase{ + Name: secret.Name, + Type: config.CommonEnvCfgTypeSecret, + EnvName: envName, + ProjectName: productName, + YamlData: string(yamlData), + Services: tempSvcs, + CreateTime: secret.GetCreationTimestamp().Time, + }, + } + resElem.setSourceDetailData(secret) + mutex.Lock() + res = append(res, resElem) + mutex.Unlock() + return nil + } + + for _, secret := range secrets { + wg.Add(1) + go func(sec *corev1.Secret) { + defer wg.Done() + err := secretProcess(sec) + if err != nil { + log.Errorf("ListSecrets ns:%s name:%s secretProcess err:%s", sec.Namespace, sec.Name, err) + } + }(secret) + } + wg.Wait() + sort.SliceStable(res, func(i, j int) bool { + return res[i].SecretName < res[j].SecretName + }) + return res, nil +} + +func UpdateSecret(args *models.CreateUpdateCommonEnvCfgArgs, userName string, log *zap.SugaredLogger) error { + js, err := yaml.YAMLToJSON([]byte(args.YamlData)) + secret := &corev1.Secret{} + err = json.Unmarshal(js, secret) + if err != nil { + return e.ErrUpdateResource.AddErr(err) + } + if secret.Name != args.Name { + return e.ErrUpdateResource.AddDesc("secret Yaml Name is incorrect") + } + + product, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: args.ProductName, + EnvName: args.EnvName, + }) + if err != nil { + return e.ErrUpdateResource.AddErr(err) + } + + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), product.ClusterID) + if err != nil { + return e.ErrUpdateResource.AddErr(err) + } + + yamlData, err := ensureLabelAndNs(secret, product.Namespace, args.ProductName) + if err != nil { + return e.ErrUpdateResource.AddErr(err) + } + + err = updater.UpdateOrCreateSecret(secret, kubeClient) + if err != nil { + log.Error(err) + return e.ErrUpdateResource.AddDesc(err.Error()) + } + envSecret := &models.EnvResource{ + ProductName: args.ProductName, + UpdateUserName: userName, + EnvName: args.EnvName, + Namespace: product.Namespace, + Name: secret.Name, + YamlData: yamlData, + Type: string(config.CommonEnvCfgTypeSecret), + SourceDetail: args.SourceDetail, + AutoSync: args.AutoSync, + } + if commonrepo.NewEnvResourceColl().Create(envSecret) != nil { + return e.ErrUpdateResource.AddDesc(err.Error()) + } + + if !args.RestartAssociatedSvc { + return nil + } + clientset, err := kubeclient.GetKubeClientSet(config.HubServerAddress(), product.ClusterID) + if err != nil { + log.Errorf("failed to create kubernetes clientset for clusterID: %s, the error is: %s", product.ClusterID, err) + return e.ErrUpdateConfigMap.AddErr(err) + } + + if err := restartPod(secret.Name, args.ProductName, args.EnvName, product.Namespace, config.CommonEnvCfgTypeSecret, clientset, kubeClient); err != nil { + return e.ErrRestartService.AddDesc(err.Error()) + } + return nil +} diff --git a/pkg/microservice/aslan/core/environment/service/service.go b/pkg/microservice/aslan/core/environment/service/service.go index 630d376478aaf4bba2daec1a57e03b2a1d872116..01eb20b4cd1271f99be73f6bb6e5bfbace962342 100644 --- a/pkg/microservice/aslan/core/environment/service/service.go +++ b/pkg/microservice/aslan/core/environment/service/service.go @@ -23,22 +23,28 @@ import ( "github.com/hashicorp/go-multierror" "go.uber.org/zap" "helm.sh/helm/v3/pkg/releaseutil" + versionedclient "istio.io/client-go/pkg/clientset/versioned" appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/informers" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" internalresource "github.com/koderover/zadig/pkg/shared/kube/resource" "github.com/koderover/zadig/pkg/shared/kube/wrapper" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/kube/getter" + "github.com/koderover/zadig/pkg/tool/kube/informer" "github.com/koderover/zadig/pkg/tool/kube/serializer" "github.com/koderover/zadig/pkg/tool/kube/updater" "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/util" ) func GetServiceContainer(envName, productName, serviceName, container string, log *zap.SugaredLogger) error { @@ -59,16 +65,11 @@ func ScaleService(envName, productName, serviceName string, number int, log *zap return e.ErrScaleService.AddErr(err) } - kubeClient, err := kube.GetKubeClient(prod.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), prod.ClusterID) if err != nil { return e.ErrScaleService.AddErr(err) } - // TODO: 目前强制定死扩容数量为0-20, 以后可以改成配置 - if number > 20 || number < 0 { - return e.ErrInvalidParam.AddDesc("伸缩数量范围[0..20]") - } - selector := labels.Set{setting.ProductLabel: productName, setting.ServiceLabel: serviceName}.AsSelector() ds, err := getter.ListDeployments(prod.Namespace, selector, kubeClient) if err != nil { @@ -100,17 +101,13 @@ func ScaleService(envName, productName, serviceName string, number int, log *zap } func Scale(args *ScaleArgs, logger *zap.SugaredLogger) error { - if args.Number > 20 || args.Number < 0 { - return e.ErrInvalidParam.AddDesc("伸缩数量范围[0..20]") - } - opt := &commonrepo.ProductFindOptions{Name: args.ProductName, EnvName: args.EnvName} prod, err := commonrepo.NewProductColl().Find(opt) if err != nil { return e.ErrScaleService.AddErr(err) } - kubeClient, err := kube.GetKubeClient(prod.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), prod.ClusterID) if err != nil { return e.ErrScaleService.AddErr(err) } @@ -140,10 +137,26 @@ func RestartScale(args *RestartScaleArgs, _ *zap.SugaredLogger) error { return err } - kubeClient, err := kube.GetKubeClient(prod.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), prod.ClusterID) + if err != nil { + return err + } + + // aws secrets needs to be refreshed + regs, err := commonservice.ListRegistryNamespaces("", true, log.SugaredLogger()) if err != nil { + log.Errorf("Failed to get registries to restart container, the error is: %s", err) return err } + for _, reg := range regs { + if reg.RegProvider == config.RegistryTypeAWS { + if err := kube.CreateOrUpdateRegistrySecret(prod.Namespace, reg, false, kubeClient); err != nil { + retErr := fmt.Errorf("failed to update pull secret for registry: %s, the error is: %s", reg.ID.Hex(), err) + log.Errorf("%s\n", retErr.Error()) + return retErr + } + } + } switch args.Type { case setting.Deployment: @@ -176,14 +189,34 @@ func GetService(envName, productName, serviceName string, workLoadType string, l return nil, e.ErrGetService.AddErr(err) } - kubeClient, err := kube.GetKubeClient(env.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), env.ClusterID) + if err != nil { + return nil, e.ErrGetService.AddErr(err) + } + + clientset, err := kubeclient.GetKubeClientSet(config.HubServerAddress(), env.ClusterID) if err != nil { + log.Errorf("Failed to create kubernetes clientset for cluster id: %s, the error is: %s", env.ClusterID, err) + return nil, e.ErrGetService.AddErr(err) + } + + inf, err := informer.NewInformer(env.ClusterID, env.Namespace, clientset) + if err != nil { + log.Errorf("Failed to create informer for namespace [%s] in cluster [%s], the error is: %s", env.Namespace, env.ClusterID, err) return nil, e.ErrGetService.AddErr(err) } namespace := env.Namespace switch env.Source { case setting.SourceFromExternal, setting.SourceFromHelm: + if workLoadType == "undefined" && env.Source == setting.SourceFromExternal { + svcOpt := &commonrepo.ServiceFindOption{ProductName: productName, ServiceName: serviceName, Type: setting.K8SDeployType} + modelSvc, err := commonrepo.NewServiceColl().Find(svcOpt) + if err != nil { + return nil, e.ErrGetService.AddErr(err) + } + workLoadType = modelSvc.WorkloadType + } k8sServices, _ := getter.ListServices(namespace, nil, kubeClient) switch workLoadType { case setting.StatefulSet: @@ -250,7 +283,12 @@ func GetService(envName, productName, serviceName string, workLoadType string, l ) } - renderSetFindOpt := &commonrepo.RenderSetFindOption{Name: env.Render.Name, Revision: env.Render.Revision} + renderSetFindOpt := &commonrepo.RenderSetFindOption{ + Name: env.Render.Name, + Revision: service.Render.Revision, + ProductTmpl: env.ProductName, + EnvName: envName, + } rs, err := commonrepo.NewRenderSetColl().Find(renderSetFindOpt) if err != nil { log.Errorf("find renderset[%s] error: %v", service.Render.Name, err) @@ -290,13 +328,37 @@ func GetService(envName, productName, serviceName string, workLoadType string, l ret.Scales = append(ret.Scales, getStatefulSetWorkloadResource(sts, kubeClient, log)) case setting.Ingress: - ing, found, err := getter.GetIngress(namespace, u.GetName(), kubeClient) - if err != nil || !found { - log.Warnf("no ingress %s found in %s:%s %v", u.GetName(), service.ServiceName, namespace, err) + + version, err := clientset.Discovery().ServerVersion() + if err != nil { + log.Warnf("Failed to determine server version, error is: %s", err) continue } + if kubeclient.VersionLessThan122(version) { + ing, found, err := getter.GetExtensionsV1Beta1Ingress(namespace, u.GetName(), inf) + if err != nil || !found { + log.Warnf("no ingress %s found in %s:%s %v", u.GetName(), service.ServiceName, namespace, err) + continue + } - ret.Ingress = append(ret.Ingress, wrapper.Ingress(ing).Resource()) + ret.Ingress = append(ret.Ingress, wrapper.Ingress(ing).Resource()) + } else { + ing, err := getter.GetNetworkingV1Ingress(namespace, u.GetName(), inf) + if err != nil { + log.Warnf("no ingress %s found in %s:%s %v", u.GetName(), service.ServiceName, namespace, err) + continue + } + + ingress := &internalresource.Ingress{ + Name: ing.Name, + Labels: ing.Labels, + HostInfo: wrapper.GetIngressHostInfo(ing), + IPs: []string{}, + Age: util.Age(ing.CreationTimestamp.Unix()), + } + + ret.Ingress = append(ret.Ingress, ingress) + } case setting.Service: svc, found, err := getter.GetService(namespace, u.GetName(), kubeClient) @@ -322,13 +384,48 @@ func RestartService(envName string, args *SvcOptArgs, log *zap.SugaredLogger) (e if err != nil { return err } - kubeClient, err := kube.GetKubeClient(productObj.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), productObj.ClusterID) + if err != nil { + return err + } + + restConfig, err := kubeclient.GetRESTConfig(config.HubServerAddress(), productObj.ClusterID) + if err != nil { + return err + } + + istioClient, err := versionedclient.NewForConfig(restConfig) + if err != nil { + return err + } + + cls, err := kubeclient.GetKubeClientSet(config.HubServerAddress(), productObj.ClusterID) + if err != nil { + return e.ErrCreateEnv.AddErr(err) + } + inf, err := informer.NewInformer(productObj.ClusterID, productObj.Namespace, cls) + if err != nil { + return e.ErrCreateEnv.AddErr(err) + } + + // aws secrets needs to be refreshed + regs, err := commonservice.ListRegistryNamespaces("", true, log) if err != nil { + log.Errorf("Failed to get registries to restart container, the error is: %s", err) return err } + for _, reg := range regs { + if reg.RegProvider == config.RegistryTypeAWS { + if err := kube.CreateOrUpdateRegistrySecret(productObj.Namespace, reg, false, kubeClient); err != nil { + retErr := fmt.Errorf("failed to update pull secret for registry: %s, the error is: %s", reg.ID.Hex(), err) + log.Errorf("%s\n", retErr.Error()) + return retErr + } + } + } switch productObj.Source { - case setting.SourceFromHelm, setting.SourceFromExternal: + case setting.SourceFromExternal: errList := new(multierror.Error) serviceObj, found, err := getter.GetService(productObj.Namespace, args.ServiceName, kubeClient) if err != nil || !found { @@ -354,7 +451,22 @@ func RestartService(envName string, args *SvcOptArgs, log *zap.SugaredLogger) (e } } } + case setting.SourceFromHelm: + deploy, found, err := getter.GetDeployment(productObj.Namespace, args.ServiceName, kubeClient) + if err != nil { + return fmt.Errorf("failed to find resource %s, type %s, err %s", args.ServiceName, setting.Deployment, err.Error()) + } + if found { + return updater.RestartDeployment(productObj.Namespace, deploy.Name, kubeClient) + } + sts, found, err := getter.GetStatefulSet(productObj.Namespace, args.ServiceName, kubeClient) + if err != nil { + return fmt.Errorf("failed to find resource %s, type %s, err %s", args.ServiceName, setting.StatefulSet, err.Error()) + } + if found { + return updater.RestartStatefulSet(productObj.Namespace, sts.Name, kubeClient) + } default: var serviceTmpl *commonmodels.Service var newRender *commonmodels.RenderSet @@ -370,7 +482,12 @@ func RestartService(envName string, args *SvcOptArgs, log *zap.SugaredLogger) (e err = e.ErrDeleteProduct.AddDesc(e.DeleteServiceContainerErrMsg + ": " + err.Error()) return } - opt := &commonrepo.RenderSetFindOption{Name: serviceObj.Render.Name, Revision: serviceObj.Render.Revision} + opt := &commonrepo.RenderSetFindOption{ + Name: serviceObj.Render.Name, + Revision: serviceObj.Render.Revision, + ProductTmpl: productObj.ProductName, + EnvName: productObj.EnvName, + } newRender, err = commonrepo.NewRenderSetColl().Find(opt) if err != nil { log.Errorf("[%s][P:%s]renderset Find error: %v", productObj.EnvName, productObj.ProductName, err) @@ -389,7 +506,7 @@ func RestartService(envName string, args *SvcOptArgs, log *zap.SugaredLogger) (e productObj, productService, productService, - newRender, kubeClient, log) + newRender, inf, kubeClient, istioClient, log) // 如果创建依赖服务组有返回错误, 停止等待 if err != nil { @@ -402,7 +519,7 @@ func RestartService(envName string, args *SvcOptArgs, log *zap.SugaredLogger) (e return nil } -func queryPodsStatus(namespace, productName, serviceName string, kubeClient client.Client, log *zap.SugaredLogger) (string, string, []string) { +func queryPodsStatus(namespace, productName, serviceName string, informer informers.SharedInformerFactory, log *zap.SugaredLogger) (string, string, []string) { ls := labels.Set{} if productName != "" { ls[setting.ProductLabel] = productName @@ -410,7 +527,7 @@ func queryPodsStatus(namespace, productName, serviceName string, kubeClient clie if serviceName != "" { ls[setting.ServiceLabel] = serviceName } - return kube.GetSelectedPodsInfo(namespace, ls.AsSelector(), kubeClient, log) + return kube.GetSelectedPodsInfo(ls.AsSelector(), informer, log) } // validateServiceContainer validate container with envName like dev @@ -421,7 +538,7 @@ func validateServiceContainer(envName, productName, serviceName, container strin return "", err } - kubeClient, err := kube.GetKubeClient(prod.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), prod.ClusterID) if err != nil { return "", err } diff --git a/pkg/microservice/aslan/core/environment/service/share_env.go b/pkg/microservice/aslan/core/environment/service/share_env.go new file mode 100644 index 0000000000000000000000000000000000000000..0047c04d4c9a1651ea62b2e760ee47391c834acb --- /dev/null +++ b/pkg/microservice/aslan/core/environment/service/share_env.go @@ -0,0 +1,1426 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "time" + + "github.com/gogo/protobuf/jsonpb" + types "github.com/gogo/protobuf/types" + helmclient "github.com/mittwald/go-helm-client" + networkingv1alpha3 "istio.io/api/networking/v1alpha3" + versionedclient "istio.io/client-go/pkg/clientset/versioned" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" + helmtool "github.com/koderover/zadig/pkg/tool/helmclient" + "github.com/koderover/zadig/pkg/tool/kube/util" + "github.com/koderover/zadig/pkg/tool/log" + zadigtypes "github.com/koderover/zadig/pkg/types" + zadigutil "github.com/koderover/zadig/pkg/util" +) + +const istioNamespace = "istio-system" +const istioProxyName = "istio-proxy" +const zadigEnvoyFilter = "zadig-share-env" +const envoyFilterNetworkHttpConnectionManager = "envoy.filters.network.http_connection_manager" +const envoyFilterHttpRouter = "envoy.filters.http.router" +const envoyFilterLua = "type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua" + +const zadigNamePrefix = "zadig" +const zadigMatchXEnv = "x-env" + +// Slice of `.` and error are returned. +func CheckWorkloadsK8sServices(ctx context.Context, envName, productName string) ([]string, error) { + timeStart := time.Now() + defer func() { + log.Infof("[CheckWorkloadsK8sServices]Time consumed: %s", time.Since(timeStart)) + }() + + prod, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: productName, + EnvName: envName, + }) + if err != nil { + return nil, fmt.Errorf("failed to query env `%s` in project `%s`: %s", envName, productName, err) + } + + ns := prod.Namespace + clusterID := prod.ClusterID + + kclient, err := kubeclient.GetKubeClient(config.HubServerAddress(), clusterID) + if err != nil { + return nil, fmt.Errorf("failed to get kube client: %s", err) + } + + return checkWorkloadsHaveK8sService(ctx, kclient, ns) +} + +func EnableBaseEnv(ctx context.Context, envName, productName string) error { + opt := &commonrepo.ProductFindOptions{Name: productName, EnvName: envName} + prod, err := commonrepo.NewProductColl().Find(opt) + if err != nil { + return fmt.Errorf("failed to query env `%s` in project `%s`: %s", envName, productName, err) + } + + ns := prod.Namespace + clusterID := prod.ClusterID + + kclient, err := kubeclient.GetKubeClient(config.HubServerAddress(), clusterID) + if err != nil { + return fmt.Errorf("failed to get kube client: %s", err) + } + + restConfig, err := kubeclient.GetRESTConfig(config.HubServerAddress(), clusterID) + if err != nil { + return fmt.Errorf("failed to get rest config: %s", err) + } + + istioClient, err := versionedclient.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("failed to new istio client: %s", err) + } + + // 1. Ensure `istio-injection=enabled` label on the namespace. + err = ensureIstioLabel(ctx, kclient, ns) + if err != nil { + return fmt.Errorf("failed to ensure istio label on namespace `%s`: %s", ns, err) + } + + // 2. Ensure Pods that are not injected with `istio-proxy`. + err = ensurePodsWithIsitoProxy(ctx, kclient, ns) + if err != nil { + return fmt.Errorf("failed to ensure pods with istio-proxy in namespace `%s`: %s", ns, err) + } + + // 3. Ensure `VirtualService` (subsets are not required) in current namespace. + err = ensureVirtualServices(ctx, prod, kclient, istioClient) + if err != nil { + return fmt.Errorf("failed to ensure VirtualServices in namespace `%s`: %s", ns, err) + } + + // 4. Ensure `EnvoyFilter` in istio namespace. + err = ensureEnvoyFilter(ctx, istioClient, clusterID, istioNamespace, zadigEnvoyFilter) + if err != nil { + return fmt.Errorf("failed to ensure EnvoyFilter in namespace `%s`: %s", istioNamespace, err) + } + + // 5. Update the environment configuration. + return ensureBaseEnvConfig(ctx, prod) +} + +func DisableBaseEnv(ctx context.Context, envName, productName string) error { + opt := &commonrepo.ProductFindOptions{Name: productName, EnvName: envName} + prod, err := commonrepo.NewProductColl().Find(opt) + if err != nil { + return fmt.Errorf("failed to query env `%s` in project `%s`: %s", envName, productName, err) + } + + ns := prod.Namespace + clusterID := prod.ClusterID + + kclient, err := kubeclient.GetKubeClient(config.HubServerAddress(), clusterID) + if err != nil { + return fmt.Errorf("failed to get kube client: %s", err) + } + + restConfig, err := kubeclient.GetRESTConfig(config.HubServerAddress(), clusterID) + if err != nil { + return fmt.Errorf("failed to get rest config: %s", err) + } + + istioClient, err := versionedclient.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("failed to new istio client: %s", err) + } + + // 1. Delete all associated subenvironments. + err = ensureDeleteAssociatedEnvs(ctx, prod) + if err != nil { + return fmt.Errorf("failed to delete associated subenvironments of base ns `%s`: %s", ns, err) + } + + // 2. Delete EnvoyFilter in the namespace of Istio installation. + err = ensureDeleteEnvoyFilter(ctx, prod, istioClient) + if err != nil { + return fmt.Errorf("failed to delete EnvoyFilter: %s", err) + } + + // 3. Delete all VirtualServices delivered by the Zadig. + err = deleteVirtualServices(ctx, kclient, istioClient, ns) + if err != nil { + return fmt.Errorf("failed to delete VirtualServices that Zadig created in ns `%s`: %s", ns, err) + } + + // 4. Remove the `istio-injection=enabled` label of the namespace. + err = removeIstioLabel(ctx, kclient, ns) + if err != nil { + return fmt.Errorf("failed to remove istio label on ns `%s`: %s", ns, err) + } + + // 5. Restart the istio-Proxy injected Pods. + err = removePodsIstioProxy(ctx, kclient, ns) + if err != nil { + return fmt.Errorf("failed to remove istio-proxy from pods in ns `%s`: %s", ns, err) + } + + // 6. Update the environment configuration. + return ensureDisableBaseEnvConfig(ctx, prod) +} + +func CheckShareEnvReady(ctx context.Context, envName, op, productName string) (*ShareEnvReady, error) { + opt := &commonrepo.ProductFindOptions{Name: productName, EnvName: envName} + prod, err := commonrepo.NewProductColl().Find(opt) + if err != nil { + return nil, fmt.Errorf("failed to query env `%s` in project `%s`: %s", envName, productName, err) + } + + ns := prod.Namespace + clusterID := prod.ClusterID + + kclient, err := kubeclient.GetKubeClient(config.HubServerAddress(), clusterID) + if err != nil { + return nil, fmt.Errorf("failed to get kube client: %s", err) + } + + restConfig, err := kubeclient.GetRESTConfig(config.HubServerAddress(), clusterID) + if err != nil { + return nil, fmt.Errorf("failed to get rest config: %s", err) + } + + istioClient, err := versionedclient.NewForConfig(restConfig) + if err != nil { + return nil, fmt.Errorf("failed to new istio client: %s", err) + } + + shareEnvOp := ShareEnvOp(op) + + // 1. Check whether namespace has labeled `istio-injection=enabled`. + isNamespaceHasIstioLabel, err := checkIstioLabel(ctx, kclient, ns) + if err != nil { + return nil, fmt.Errorf("failed to check whether namespace `%s` has labeled `istio-injection=enabled`: %s", ns, err) + } + + // 2. Check whether all workloads have K8s Service. + workloadsHaveNoK8sService, err := checkWorkloadsHaveK8sService(ctx, kclient, ns) + if err != nil { + return nil, fmt.Errorf("failed to check whether all workloads in ns `%s` have K8s Service: %s", ns, err) + } + + isWorkloadsHaveNoK8sService := true + if len(workloadsHaveNoK8sService) > 0 { + isWorkloadsHaveNoK8sService = false + } + + // 3. Check whether all VirtualServices have been deployed. + isVirtualServicesDeployed, err := checkVirtualServicesDeployed(ctx, kclient, istioClient, ns) + if err != nil { + return nil, fmt.Errorf("failed to check whether all VirtualServices in ns `%s` have been deployed: %s", ns, err) + } + + // 4. Check whether all Pods have istio-proxy and are ready. + allHaveIstioProxy, allPodsReady, err := checkPodsWithIstioProxyAndReady(ctx, kclient, ns) + if err != nil { + return nil, fmt.Errorf("failed to check whether all pods in ns `%s` have istio-proxy and are ready: %s", ns, err) + } + + res := &ShareEnvReady{ + Checks: ShareEnvReadyChecks{ + NamespaceHasIstioLabel: isNamespaceHasIstioLabel, + WorkloadsHaveK8sService: isWorkloadsHaveNoK8sService, + VirtualServicesDeployed: isVirtualServicesDeployed, + PodsHaveIstioProxy: allHaveIstioProxy, + WorkloadsReady: allPodsReady, + }, + } + res.CheckAndSetReady(shareEnvOp) + + return res, nil +} + +func checkWorkloadsHaveK8sService(ctx context.Context, kclient client.Client, ns string) ([]string, error) { + workloads, err := getWorkloads(ctx, kclient, ns) + if err != nil { + return nil, fmt.Errorf("failed to get workloads: %s", err) + } + + svcs, err := getSvcs(ctx, kclient, ns) + if err != nil { + return nil, fmt.Errorf("failed to get svcs: %s", err) + } + + return checkWorkloadsNoSvcs(svcs, workloads) +} + +// map structure: +// - key: `.`, e.g.: zadig.deployment +// - value: a map[string]string, which is workload' labels +// +// Note: +// Currently we only support Deployment and StatefulSet workloads, if more workloads need to be supported, +// they need to be added here. +func getWorkloads(ctx context.Context, kclient client.Client, namespace string) (map[string]map[string]string, error) { + deployments := &appsv1.DeploymentList{} + err := kclient.List(ctx, deployments, client.InNamespace(namespace)) + if err != nil { + return nil, fmt.Errorf("failed to list deployments in %s: %s", namespace, err) + } + + statefulsets := &appsv1.StatefulSetList{} + err = kclient.List(ctx, statefulsets, client.InNamespace(namespace)) + if err != nil { + return nil, fmt.Errorf("failed to list statefulsets in %s: %s", namespace, err) + } + + res := map[string]map[string]string{} + for _, d := range deployments.Items { + res[fmt.Sprintf("%s.%s", d.Name, d.Kind)] = d.Spec.Template.GetLabels() + } + for _, s := range statefulsets.Items { + res[fmt.Sprintf("%s.%s", s.Name, s.Kind)] = s.Spec.Template.GetLabels() + } + + return res, nil +} + +// map structure: +// - key: ``, e.g.: zadig +// - value: a map[string]string, which is Service's selector +func getSvcs(ctx context.Context, kclient client.Client, namespace string) (map[string]map[string]string, error) { + svcs := &corev1.ServiceList{} + err := kclient.List(ctx, svcs, client.InNamespace(namespace)) + if err != nil { + return nil, fmt.Errorf("failed to list svcs in %s: %s", namespace, err) + } + + res := map[string]map[string]string{} + for _, s := range svcs.Items { + res[s.Name] = s.Spec.Selector + } + + return res, nil +} + +// Structure of `svcs` and `workloads` is same as the above. +// Slice of `.` and error are returned. +func checkWorkloadsNoSvcs(svcs map[string]map[string]string, workloads map[string]map[string]string) ([]string, error) { + for _, svcSelector := range svcs { + for wName, wLabels := range workloads { + if !isMapSubset(svcSelector, wLabels) { + continue + } + + delete(workloads, wName) + + // Note: Generally, a workload does not correspond to multiple services in a namespace. + break + } + } + + workloadsName := make([]string, 0, len(workloads)) + for wname := range workloads { + workloadsName = append(workloadsName, wname) + } + + return workloadsName, nil +} + +func isMapSubset(mapSubSet, mapSet map[string]string) bool { + if len(mapSubSet) > len(mapSet) { + return false + } + + for k, v := range mapSubSet { + if vv, found := mapSet[k]; !found || vv != v { + return false + } + } + + return true +} + +func ensureIstioLabel(ctx context.Context, kclient client.Client, ns string) error { + nsObj := &corev1.Namespace{} + err := kclient.Get(ctx, client.ObjectKey{ + Name: ns, + }, nsObj) + if err != nil { + return fmt.Errorf("failed to query ns `%s`: %s", ns, err) + } + + nsObj.Labels[zadigtypes.IstioLabelKeyInjection] = zadigtypes.IstioLabelValueInjection + return kclient.Update(ctx, nsObj) +} + +func checkIstioLabel(ctx context.Context, kclient client.Client, ns string) (bool, error) { + nsObj := &corev1.Namespace{} + err := kclient.Get(ctx, client.ObjectKey{ + Name: ns, + }, nsObj) + if err != nil { + return false, fmt.Errorf("failed to query ns `%s`: %s", ns, err) + } + + return nsObj.Labels[zadigtypes.IstioLabelKeyInjection] == zadigtypes.IstioLabelValueInjection, nil +} + +func ensurePodsWithIsitoProxy(ctx context.Context, kclient client.Client, ns string) error { + return restartPodsWithIstioProxy(ctx, kclient, ns, false) +} + +func checkPodsWithIstioProxyAndReady(ctx context.Context, kclient client.Client, ns string) (allHaveIstioProxy, allPodsReady bool, err error) { + pods := &corev1.PodList{} + err = kclient.List(ctx, pods, client.InNamespace(ns)) + if err != nil { + return false, false, fmt.Errorf("failed to query pods in ns `%s`: %s", ns, err) + } + + allHaveIstioProxy = true + allPodsReady = true + for _, pod := range pods.Items { + if !(pod.Status.Phase == corev1.PodRunning || pod.Status.Phase == corev1.PodSucceeded) { + allPodsReady = false + } + + hasIstioProxy := false + for _, container := range pod.Spec.Containers { + if container.Name == istioProxyName { + hasIstioProxy = true + break + } + } + if !hasIstioProxy { + allHaveIstioProxy = false + } + + if !allPodsReady && !allHaveIstioProxy { + return false, false, nil + } + } + + return allHaveIstioProxy, allPodsReady, nil +} + +func ensureVirtualServices(ctx context.Context, env *commonmodels.Product, kclient client.Client, istioClient versionedclient.Interface) error { + svcs := &corev1.ServiceList{} + err := kclient.List(ctx, svcs, client.InNamespace(env.Namespace)) + if err != nil { + return fmt.Errorf("failed to list svcs in ns `%s`: %s", env.Namespace, err) + } + + for _, svc := range svcs.Items { + vsName := genVirtualServiceName(&svc) + err := ensureVirtualService(ctx, kclient, istioClient, env, &svc, vsName) + if err != nil { + return fmt.Errorf("failed to ensure VirtualService `%s` in ns `%s`: %s", vsName, env.Namespace, err) + } + } + + return nil +} + +func ensureVirtualService(ctx context.Context, kclient client.Client, istioClient versionedclient.Interface, env *commonmodels.Product, svc *corev1.Service, vsName string) error { + vsObj, err := istioClient.NetworkingV1alpha3().VirtualServices(env.Namespace).Get(ctx, vsName, metav1.GetOptions{}) + if err == nil { + log.Infof("Has found VirtualService `%s` in ns `%s` and don't recreate.", vsName, env.Namespace) + return nil + } + + if !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to query VirtualService `%s` in ns `%s`: %s", vsName, env.Namespace, err) + } + + matchedEnvs := []MatchedEnv{} + if env.ShareEnv.Enable && env.ShareEnv.IsBase { + subEnvs, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{ + Name: env.ProductName, + ShareEnvEnable: zadigutil.GetBoolPointer(true), + ShareEnvIsBase: zadigutil.GetBoolPointer(false), + ShareEnvBaseEnv: zadigutil.GetStrPointer(env.EnvName), + }) + if err != nil { + return err + } + + svcSelector := labels.SelectorFromSet(labels.Set(svc.Spec.Selector)) + for _, subEnv := range subEnvs { + hasWorkload, err := doesSvcHasWorkload(ctx, subEnv.Namespace, svcSelector, kclient) + if err != nil { + return err + } + + if !hasWorkload { + continue + } + + matchedEnvs = append(matchedEnvs, MatchedEnv{ + EnvName: subEnv.EnvName, + Namespace: subEnv.Namespace, + }) + } + } + + vsObj.Name = vsName + + if vsObj.Labels == nil { + vsObj.Labels = map[string]string{} + } + vsObj.Labels[zadigtypes.ZadigLabelKeyGlobalOwner] = zadigtypes.Zadig + + routes := []*networkingv1alpha3.HTTPRoute{} + for _, matchedEnv := range matchedEnvs { + grayRoute := &networkingv1alpha3.HTTPRoute{ + Match: []*networkingv1alpha3.HTTPMatchRequest{ + &networkingv1alpha3.HTTPMatchRequest{ + Headers: map[string]*networkingv1alpha3.StringMatch{ + zadigMatchXEnv: &networkingv1alpha3.StringMatch{ + MatchType: &networkingv1alpha3.StringMatch_Exact{ + Exact: matchedEnv.EnvName, + }, + }, + }, + }, + }, + Route: []*networkingv1alpha3.HTTPRouteDestination{ + &networkingv1alpha3.HTTPRouteDestination{ + Destination: &networkingv1alpha3.Destination{ + Host: fmt.Sprintf("%s.%s.svc.cluster.local", svc.Name, matchedEnv.Namespace), + }, + }, + }, + } + routes = append(routes, grayRoute) + } + routes = append(routes, &networkingv1alpha3.HTTPRoute{ + Route: []*networkingv1alpha3.HTTPRouteDestination{ + &networkingv1alpha3.HTTPRouteDestination{ + Destination: &networkingv1alpha3.Destination{ + Host: fmt.Sprintf("%s.%s.svc.cluster.local", svc.Name, env.Namespace), + }, + }, + }, + }) + + vsObj.Spec = networkingv1alpha3.VirtualService{ + Hosts: []string{svc.Name}, + Http: routes, + } + _, err = istioClient.NetworkingV1alpha3().VirtualServices(env.Namespace).Create(ctx, vsObj, metav1.CreateOptions{}) + return err +} + +func checkVirtualServicesDeployed(ctx context.Context, kclient client.Client, istioClient versionedclient.Interface, ns string) (bool, error) { + svcs := &corev1.ServiceList{} + err := kclient.List(ctx, svcs, client.InNamespace(ns)) + if err != nil { + return false, fmt.Errorf("failed to list svcs in ns `%s`: %s", ns, err) + } + + for _, svc := range svcs.Items { + vsName := genVirtualServiceName(&svc) + _, err := istioClient.NetworkingV1alpha3().VirtualServices(ns).Get(ctx, vsName, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("failed to query VirtualService `%s` in ns `%s`: %s", vsName, ns, err) + } + } + + return true, nil +} + +func ensureEnvoyFilter(ctx context.Context, istioClient versionedclient.Interface, clusterID, ns, name string) error { + envoyFilterObj, err := istioClient.NetworkingV1alpha3().EnvoyFilters(ns).Get(ctx, name, metav1.GetOptions{}) + if err == nil { + log.Infof("Has found EnvoyFilter `%s` in ns `%s` and don't recreate.", name, istioNamespace) + return nil + } + + if !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to query EnvoyFilter `%s` in ns `%s`: %s", name, istioNamespace, err) + } + + storeCacheOperation, err := buildEnvoyStoreCacheOperation() + if err != nil { + return fmt.Errorf("failed to build envoy operation of storing cache: %s", err) + } + + getCacheOperation, err := buildEnvoyGetCacheOperation() + if err != nil { + return fmt.Errorf("failed to build envoy operation of getting cache: %s", err) + } + + var cacheServerAddr string + var cacheServerPort int + + switch clusterID { + case setting.LocalClusterID: + cacheServerAddr = fmt.Sprintf("aslan.%s.svc.cluster.local", config.Namespace()) + cacheServerPort = 25000 + default: + cacheServerAddr = "hub-agent.koderover-agent.svc.cluster.local" + cacheServerPort = 80 + } + + clusterConfig, err := buildEnvoyClusterConfig(cacheServerAddr, cacheServerPort) + if err != nil { + return fmt.Errorf("failed to build envoy cluster config for `%s:%d`: %s", cacheServerAddr, cacheServerPort, err) + } + + envoyFilterObj.Name = name + + if envoyFilterObj.Labels == nil { + envoyFilterObj.Labels = map[string]string{} + } + envoyFilterObj.Labels[zadigtypes.ZadigLabelKeyGlobalOwner] = zadigtypes.Zadig + + envoyFilterObj.Spec = networkingv1alpha3.EnvoyFilter{ + ConfigPatches: []*networkingv1alpha3.EnvoyFilter_EnvoyConfigObjectPatch{ + &networkingv1alpha3.EnvoyFilter_EnvoyConfigObjectPatch{ + ApplyTo: networkingv1alpha3.EnvoyFilter_HTTP_FILTER, + Match: &networkingv1alpha3.EnvoyFilter_EnvoyConfigObjectMatch{ + Context: networkingv1alpha3.EnvoyFilter_SIDECAR_INBOUND, + ObjectTypes: &networkingv1alpha3.EnvoyFilter_EnvoyConfigObjectMatch_Listener{ + Listener: &networkingv1alpha3.EnvoyFilter_ListenerMatch{ + FilterChain: &networkingv1alpha3.EnvoyFilter_ListenerMatch_FilterChainMatch{ + Filter: &networkingv1alpha3.EnvoyFilter_ListenerMatch_FilterMatch{ + Name: envoyFilterNetworkHttpConnectionManager, + SubFilter: &networkingv1alpha3.EnvoyFilter_ListenerMatch_SubFilterMatch{ + Name: envoyFilterHttpRouter, + }, + }, + }, + }, + }, + }, + Patch: &networkingv1alpha3.EnvoyFilter_Patch{ + Operation: networkingv1alpha3.EnvoyFilter_Patch_INSERT_BEFORE, + Value: storeCacheOperation, + }, + }, + &networkingv1alpha3.EnvoyFilter_EnvoyConfigObjectPatch{ + ApplyTo: networkingv1alpha3.EnvoyFilter_HTTP_FILTER, + Match: &networkingv1alpha3.EnvoyFilter_EnvoyConfigObjectMatch{ + Context: networkingv1alpha3.EnvoyFilter_SIDECAR_OUTBOUND, + ObjectTypes: &networkingv1alpha3.EnvoyFilter_EnvoyConfigObjectMatch_Listener{ + Listener: &networkingv1alpha3.EnvoyFilter_ListenerMatch{ + FilterChain: &networkingv1alpha3.EnvoyFilter_ListenerMatch_FilterChainMatch{ + Filter: &networkingv1alpha3.EnvoyFilter_ListenerMatch_FilterMatch{ + Name: envoyFilterNetworkHttpConnectionManager, + SubFilter: &networkingv1alpha3.EnvoyFilter_ListenerMatch_SubFilterMatch{ + Name: envoyFilterHttpRouter, + }, + }, + }, + }, + }, + }, + Patch: &networkingv1alpha3.EnvoyFilter_Patch{ + Operation: networkingv1alpha3.EnvoyFilter_Patch_INSERT_BEFORE, + Value: getCacheOperation, + }, + }, + &networkingv1alpha3.EnvoyFilter_EnvoyConfigObjectPatch{ + ApplyTo: networkingv1alpha3.EnvoyFilter_CLUSTER, + Patch: &networkingv1alpha3.EnvoyFilter_Patch{ + Operation: networkingv1alpha3.EnvoyFilter_Patch_ADD, + Value: clusterConfig, + }, + }, + }, + } + + _, err = istioClient.NetworkingV1alpha3().EnvoyFilters(ns).Create(ctx, envoyFilterObj, metav1.CreateOptions{}) + return err +} + +func buildEnvoyStoreCacheOperation() (*types.Struct, error) { + inlineCode := `function envoy_on_request(request_handle) + function split_str(s, delimiter) + res = {} + for match in (s..delimiter):gmatch("(.-)"..delimiter) do + table.insert(res, match) + end + + return res + end + + local traceid = request_handle:headers():get("sw8") + if traceid then + arr = split_str(traceid, "-") + traceid = arr[2] + else + traceid = request_handle:headers():get("x-request-id") + if not traceid then + traceid = request_handle:headers():get("x-b3-traceid") + end + end + + local env = request_handle:headers():get("x-env") + local headers, body = request_handle:httpCall( + "cache", + { + [":method"] = "POST", + [":path"] = string.format("/api/cache/%s/%s", traceid, env), + [":authority"] = "cache", + }, + "", + 5000 + ) +end +` + data := map[string]interface{}{ + "name": "envoy.lua", + "typed_config": map[string]string{ + "@type": envoyFilterLua, + "inlineCode": inlineCode, + }, + } + + return buildEnvoyPatchValue(data) +} + +func buildEnvoyGetCacheOperation() (*types.Struct, error) { + inlineCode := `function envoy_on_request(request_handle) + function split_str(s, delimiter) + res = {} + for match in (s..delimiter):gmatch("(.-)"..delimiter) do + table.insert(res, match) + end + + return res + end + + local traceid = request_handle:headers():get("sw8") + if traceid then + arr = split_str(traceid, "-") + traceid = arr[2] + else + traceid = request_handle:headers():get("x-request-id") + if not traceid then + traceid = request_handle:headers():get("x-b3-traceid") + end + end + + local headers, body = request_handle:httpCall( + "cache", + { + [":method"] = "GET", + [":path"] = string.format("/api/cache/%s", traceid), + [":authority"] = "cache", + }, + "", + 5000 + ) + + request_handle:headers():add("x-env", headers["x-data"]); +end +` + data := map[string]interface{}{ + "name": "envoy.lua", + "typed_config": map[string]string{ + "@type": envoyFilterLua, + "inlineCode": inlineCode, + }, + } + + return buildEnvoyPatchValue(data) +} + +func buildEnvoyClusterConfig(cacheAddr string, port int) (*types.Struct, error) { + data := map[string]interface{}{ + "name": "cache", + "type": "STRICT_DNS", + "connect_timeout": "3.0s", + "lb_policy": "ROUND_ROBIN", + "load_assignment": EnvoyClusterConfigLoadAssignment{ + ClusterName: "cache", + Endpoints: []EnvoyLBEndpoints{ + EnvoyLBEndpoints{ + LBEndpoints: []EnvoyEndpoints{ + EnvoyEndpoints{ + Endpoint: EnvoyEndpoint{ + Address: EnvoyAddress{ + SocketAddress: EnvoySocketAddress{ + Protocol: "TCP", + Address: cacheAddr, + PortValue: port, + }, + }, + }, + }, + }, + }, + }, + }, + } + + return buildEnvoyPatchValue(data) +} + +func buildEnvoyPatchValue(data map[string]interface{}) (*types.Struct, error) { + dataBytes, err := json.Marshal(data) + if err != nil { + return nil, fmt.Errorf("failed to marshal data: %s", err) + } + + reader := bytes.NewReader(dataBytes) + val := &types.Struct{} + err = jsonpb.Unmarshal(reader, val) + return val, err +} + +func deleteVirtualServices(ctx context.Context, kclient client.Client, istioClient versionedclient.Interface, ns string) error { + zadigLabels := map[string]string{ + zadigtypes.ZadigLabelKeyGlobalOwner: zadigtypes.Zadig, + } + + vsObjs, err := istioClient.NetworkingV1alpha3().VirtualServices(ns).List(ctx, metav1.ListOptions{ + LabelSelector: labels.FormatLabels(zadigLabels), + }) + if err != nil { + return fmt.Errorf("failed to list VirtualServices in ns `%s`: %s", ns, err) + } + + deleteOption := metav1.DeletePropagationBackground + for _, vsObj := range vsObjs.Items { + err := istioClient.NetworkingV1alpha3().VirtualServices(ns).Delete(ctx, vsObj.Name, metav1.DeleteOptions{ + PropagationPolicy: &deleteOption, + }) + if err != nil { + return fmt.Errorf("failed to delete VirtualService %s in ns `%s`: %s", vsObj.Name, ns, err) + } + } + + return nil +} + +func removeIstioLabel(ctx context.Context, kclient client.Client, ns string) error { + nsObj := &corev1.Namespace{} + err := kclient.Get(ctx, client.ObjectKey{ + Name: ns, + }, nsObj) + if err != nil { + return fmt.Errorf("failed to query ns `%s`: %s", ns, err) + } + + delete(nsObj.Labels, zadigtypes.IstioLabelKeyInjection) + return kclient.Update(ctx, nsObj) +} + +func removePodsIstioProxy(ctx context.Context, kclient client.Client, ns string) error { + return restartPodsWithIstioProxy(ctx, kclient, ns, true) +} + +func genVirtualServiceName(svc *corev1.Service) string { + return fmt.Sprintf("%s-%s", zadigNamePrefix, svc.Name) +} + +func restartPodsWithIstioProxy(ctx context.Context, kclient client.Client, ns string, restartConditionHasIstioProxy bool) error { + pods := &corev1.PodList{} + err := kclient.List(ctx, pods, client.InNamespace(ns)) + if err != nil { + return fmt.Errorf("failed to query pods in ns `%s`: %s", ns, err) + } + + deleteOption := metav1.DeletePropagationBackground + for _, pod := range pods.Items { + hasIstioProxy := false + for _, container := range pod.Spec.Containers { + if container.Name == istioProxyName { + hasIstioProxy = true + break + } + } + + if hasIstioProxy == restartConditionHasIstioProxy { + err := kclient.Delete(ctx, &pod, &client.DeleteOptions{ + PropagationPolicy: &deleteOption, + }) + + if err != nil { + return fmt.Errorf("failed to delete Pod `%s` in ns `%s`: %s", pod.Name, ns, err) + } + } + } + + return nil +} + +func EnsureGrayEnvConfig(ctx context.Context, env *commonmodels.Product, kclient client.Client, istioClient versionedclient.Interface) error { + opt := &commonrepo.ProductFindOptions{Name: env.ProductName, EnvName: env.ShareEnv.BaseEnv} + baseEnv, err := commonrepo.NewProductColl().Find(opt) + if err != nil { + return fmt.Errorf("failed to find base env %s of product %s: %s", env.EnvName, env.ProductName, err) + } + + baseNS := baseEnv.Namespace + + // 1. Deploy VirtualServices of the workloads in gray environment and update them in the base environment. + err = ensureWorkloadsVirtualServiceInGrayAndBase(ctx, env, baseNS, kclient, istioClient) + if err != nil { + return fmt.Errorf("failed to ensure workloads VirtualService: %s", err) + } + + // 2. Deploy K8s Services and VirtualServices of all workloads in the base environment to the gray environment. + err = ensureDefaultK8sServiceAndVirtualServicesInGray(ctx, env, baseNS, kclient, istioClient) + if err != nil { + return fmt.Errorf("failed to ensure K8s Services and VirtualServices: %s", err) + } + + return nil +} + +func ensureDefaultK8sServiceAndVirtualServicesInGray(ctx context.Context, env *commonmodels.Product, baseNS string, kclient client.Client, istioClient versionedclient.Interface) error { + svcsInBase := &corev1.ServiceList{} + err := kclient.List(ctx, svcsInBase, client.InNamespace(baseNS)) + if err != nil { + return fmt.Errorf("failed to list svcs in %s: %s", baseNS, err) + } + + grayNS := env.Namespace + for _, svcInBase := range svcsInBase.Items { + err = ensureDefaultK8sServiceInGray(ctx, &svcInBase, grayNS, kclient) + if err != nil { + return err + } + + err = ensureDefaultVirtualServiceInGray(ctx, &svcInBase, grayNS, baseNS, istioClient) + if err != nil { + return err + } + } + + return nil +} + +func ensureDefaultK8sServiceInGray(ctx context.Context, baseSvc *corev1.Service, grayNS string, kclient client.Client) error { + svcInGray := &corev1.Service{} + err := kclient.Get(ctx, client.ObjectKey{ + Name: baseSvc.Name, + Namespace: grayNS, + }, svcInGray) + if err == nil { + return nil + } + if !apierrors.IsNotFound(err) { + return err + } + + svcInGray.Name = baseSvc.Name + svcInGray.Namespace = grayNS + svcInGray.Labels = baseSvc.Labels + svcInGray.Annotations = baseSvc.Annotations + svcInGray.Spec.Selector = baseSvc.Spec.Selector + + ports := make([]corev1.ServicePort, len(baseSvc.Spec.Ports)) + for i, port := range baseSvc.Spec.Ports { + ports[i] = corev1.ServicePort{ + Name: port.Name, + Protocol: port.Protocol, + Port: port.Port, + TargetPort: port.TargetPort, + } + } + svcInGray.Spec.Ports = ports + + if svcInGray.Labels != nil { + svcInGray.Labels[zadigtypes.ZadigLabelKeyGlobalOwner] = zadigtypes.Zadig + } + + return kclient.Create(ctx, svcInGray) +} + +func ensureDefaultVirtualServiceInGray(ctx context.Context, baseSvc *corev1.Service, grayNS, baseNS string, istioClient versionedclient.Interface) error { + vsName := genVirtualServiceName(baseSvc) + svcName := baseSvc.Name + vsObj, err := istioClient.NetworkingV1alpha3().VirtualServices(grayNS).Get(ctx, vsName, metav1.GetOptions{}) + if err == nil { + log.Infof("Has found VirtualService `%s` in ns `%s` and don't recreate.", vsName, grayNS) + return nil + } + + if !apierrors.IsNotFound(err) { + return err + } + + vsObj.Name = vsName + + if vsObj.Labels == nil { + vsObj.Labels = map[string]string{} + } + vsObj.Labels[zadigtypes.ZadigLabelKeyGlobalOwner] = zadigtypes.Zadig + + vsObj.Spec = networkingv1alpha3.VirtualService{ + Hosts: []string{svcName}, + Http: []*networkingv1alpha3.HTTPRoute{ + &networkingv1alpha3.HTTPRoute{ + Route: []*networkingv1alpha3.HTTPRouteDestination{ + &networkingv1alpha3.HTTPRouteDestination{ + Destination: &networkingv1alpha3.Destination{ + Host: fmt.Sprintf("%s.%s.svc.cluster.local", svcName, baseNS), + }, + }, + }, + }, + }, + } + _, err = istioClient.NetworkingV1alpha3().VirtualServices(grayNS).Create(ctx, vsObj, metav1.CreateOptions{}) + return err +} + +// Note: Currently we have made an assumption that all the necessary K8s services exist in the current environment. +func ensureWorkloadsVirtualServiceInGrayAndBase(ctx context.Context, env *commonmodels.Product, baseNS string, kclient client.Client, istioClient versionedclient.Interface) error { + svcs := &corev1.ServiceList{} + grayNS := env.Namespace + err := kclient.List(ctx, svcs, client.InNamespace(grayNS)) + if err != nil { + return err + } + + for _, svc := range svcs.Items { + // If there is no workloads in the sub-environment, the service is not updated. + hasWorkload, err := doesSvcHasWorkload(ctx, grayNS, labels.SelectorFromSet(labels.Set(svc.Spec.Selector)), kclient) + if err != nil { + return err + } + + if !hasWorkload { + continue + } + + vsName := genVirtualServiceName(&svc) + + err = ensureVirtualServiceInGray(ctx, env.EnvName, vsName, svc.Name, grayNS, baseNS, istioClient) + if err != nil { + return err + } + + err = ensureUpdateVirtualServiceInBase(ctx, env.EnvName, vsName, svc.Name, grayNS, baseNS, istioClient) + if err != nil { + return err + } + } + + return nil +} + +func ensureVirtualServiceInGray(ctx context.Context, envName, vsName, svcName, grayNS, baseNS string, istioClient versionedclient.Interface) error { + var isExisted bool + + vsObjInGray, err := istioClient.NetworkingV1alpha3().VirtualServices(grayNS).Get(ctx, vsName, metav1.GetOptions{}) + if err == nil { + isExisted = true + } + if err != nil && !apierrors.IsNotFound(err) { + return err + } + + vsObjInGray.Name = vsName + vsObjInGray.Namespace = grayNS + + if vsObjInGray.Labels == nil { + vsObjInGray.Labels = map[string]string{} + } + vsObjInGray.Labels[zadigtypes.ZadigLabelKeyGlobalOwner] = zadigtypes.Zadig + + vsObjInGray.Spec = networkingv1alpha3.VirtualService{ + Hosts: []string{svcName}, + Http: []*networkingv1alpha3.HTTPRoute{ + &networkingv1alpha3.HTTPRoute{ + Match: []*networkingv1alpha3.HTTPMatchRequest{ + &networkingv1alpha3.HTTPMatchRequest{ + Headers: map[string]*networkingv1alpha3.StringMatch{ + zadigMatchXEnv: &networkingv1alpha3.StringMatch{ + MatchType: &networkingv1alpha3.StringMatch_Exact{ + Exact: envName, + }, + }, + }, + }, + }, + Route: []*networkingv1alpha3.HTTPRouteDestination{ + &networkingv1alpha3.HTTPRouteDestination{ + Destination: &networkingv1alpha3.Destination{ + Host: fmt.Sprintf("%s.%s.svc.cluster.local", svcName, grayNS), + }, + }, + }, + }, + &networkingv1alpha3.HTTPRoute{ + Route: []*networkingv1alpha3.HTTPRouteDestination{ + &networkingv1alpha3.HTTPRouteDestination{ + Destination: &networkingv1alpha3.Destination{ + Host: fmt.Sprintf("%s.%s.svc.cluster.local", svcName, baseNS), + }, + }, + }, + }, + }, + } + + if isExisted { + _, err = istioClient.NetworkingV1alpha3().VirtualServices(grayNS).Update(ctx, vsObjInGray, metav1.UpdateOptions{}) + } else { + _, err = istioClient.NetworkingV1alpha3().VirtualServices(grayNS).Create(ctx, vsObjInGray, metav1.CreateOptions{}) + } + + return err +} + +func ensureUpdateVirtualServiceInBase(ctx context.Context, envName, vsName, svcName, grayNS, baseNS string, istioClient versionedclient.Interface) error { + vsObjInBase, err := istioClient.NetworkingV1alpha3().VirtualServices(baseNS).Get(ctx, vsName, metav1.GetOptions{}) + if err != nil { + return err + } + + if vsObjInBase.Spec.Http == nil { + vsObjInBase.Spec.Http = []*networkingv1alpha3.HTTPRoute{} + } + + for _, vsHttp := range vsObjInBase.Spec.Http { + if len(vsHttp.Match) == 0 { + continue + } + + for _, vsMatch := range vsHttp.Match { + if len(vsMatch.Headers) == 0 { + continue + } + + matchValue, found := vsMatch.Headers[zadigMatchXEnv] + if !found { + continue + } + + if matchValue.GetExact() == envName { + return nil + } + } + } + + grayRoute := &networkingv1alpha3.HTTPRoute{ + Match: []*networkingv1alpha3.HTTPMatchRequest{ + &networkingv1alpha3.HTTPMatchRequest{ + Headers: map[string]*networkingv1alpha3.StringMatch{ + zadigMatchXEnv: &networkingv1alpha3.StringMatch{ + MatchType: &networkingv1alpha3.StringMatch_Exact{ + Exact: envName, + }, + }, + }, + }, + }, + Route: []*networkingv1alpha3.HTTPRouteDestination{ + &networkingv1alpha3.HTTPRouteDestination{ + Destination: &networkingv1alpha3.Destination{ + Host: fmt.Sprintf("%s.%s.svc.cluster.local", svcName, grayNS), + }, + }, + }, + } + + numRoutes := len(vsObjInBase.Spec.Http) + if numRoutes == 0 { + vsObjInBase.Spec.Http = append(vsObjInBase.Spec.Http, grayRoute) + } else { + routes := make([]*networkingv1alpha3.HTTPRoute, 1, numRoutes+1) + routes[0] = grayRoute + routes = append(routes, vsObjInBase.Spec.Http...) + vsObjInBase.Spec.Http = routes + } + + _, err = istioClient.NetworkingV1alpha3().VirtualServices(baseNS).Update(ctx, vsObjInBase, metav1.UpdateOptions{}) + return err +} + +func EnsureDeleteShareEnvConfig(ctx context.Context, env *commonmodels.Product, istioClient versionedclient.Interface) error { + if !env.ShareEnv.Enable { + return nil + } + + if env.ShareEnv.IsBase { + // 1. Delete all associated subenvironments. + err := ensureDeleteAssociatedEnvs(ctx, env) + if err != nil { + return err + } + + // 2. If no environment is shared, delete EnvoyFilter. + return ensureDeleteEnvoyFilter(ctx, env, istioClient) + } + + // Updated the VirtualService configuration in the base environment. + return ensureCleanRoutesInBase(ctx, env, istioClient) +} + +func EnsureUpdateZadigService(ctx context.Context, env *commonmodels.Product, svcName string, kclient client.Client, istioClient versionedclient.Interface) error { + if !env.ShareEnv.Enable { + return nil + } + + // Note: A Service may not be queried immediately after it is created. + var err error + svc := &corev1.Service{} + for i := 0; i < 3; i++ { + err = kclient.Get(ctx, client.ObjectKey{ + Name: svcName, + Namespace: env.Namespace, + }, svc) + if err == nil { + break + } + + log.Warnf("Failed to query Service %s in ns %s: %s", svcName, env.Namespace, err) + time.Sleep(1 * time.Second) + } + if err != nil { + return fmt.Errorf("failed to query Service %s in ns %s: %s", svcName, env.Namespace, err) + } + + return ensureUpdateZadigSerivce(ctx, env, svc, kclient, istioClient) +} + +func EnsureDeleteZadigService(ctx context.Context, env *commonmodels.Product, svcSelector labels.Selector, kclient client.Client, istioClient versionedclient.Interface) error { + if !env.ShareEnv.Enable { + return nil + } + + svcList := &corev1.ServiceList{} + err := kclient.List(ctx, svcList, &client.ListOptions{ + Namespace: env.Namespace, + LabelSelector: svcSelector, + }) + if err != nil { + return util.IgnoreNotFoundError(err) + } + + if len(svcList.Items) != 1 { + return fmt.Errorf("Length of svc list is not expected for env %s of product %s with selector %s. Expected 1 but got %d.", env.EnvName, env.ProductName, svcSelector.String(), len(svcList.Items)) + } + svc := &svcList.Items[0] + + return ensureDeleteZadigService(ctx, env, svc, kclient, istioClient) +} + +func GetEnvServiceList(ctx context.Context, productName, baseEnvName string) ([][]string, error) { + env, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: productName, + EnvName: baseEnvName, + }) + if err != nil { + return nil, fmt.Errorf("failed to query env `%s` in project `%s`: %s", baseEnvName, productName, err) + } + + svcGroupNames := make([][]string, len(env.Services)) + for i, svcArr := range env.Services { + svcGroupNames[i] = []string{} + for _, svc := range svcArr { + svcGroupNames[i] = append(svcGroupNames[i], svc.ServiceName) + } + } + + return svcGroupNames, nil +} + +// Map of map[ServiceName][]string{EnvName} is returned. +func CheckServicesDeployedInSubEnvs(ctx context.Context, productName, envName string, services []string) (map[string][]string, error) { + env, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: productName, + EnvName: envName, + }) + if err != nil { + return nil, fmt.Errorf("failed to query env %s of product %s: %s", envName, productName, err) + } + if !env.ShareEnv.Enable || !env.ShareEnv.IsBase { + return nil, nil + } + + envs, err := fetchSubEnvs(ctx, productName, env.ClusterID, envName) + if err != nil { + return nil, err + } + + envsSvcs := make(map[string][]string, len(envs)) + for _, env := range envs { + envsSvcs[env.EnvName] = getSvcInEnv(env) + } + + svcsInSubEnvs := map[string][]string{} + for _, svcName := range services { + for envName, svcNames := range envsSvcs { + if !zadigutil.InStringArray(svcName, svcNames) { + continue + } + + if _, found := svcsInSubEnvs[svcName]; !found { + svcsInSubEnvs[svcName] = []string{} + } + svcsInSubEnvs[svcName] = append(svcsInSubEnvs[svcName], envName) + } + } + + return svcsInSubEnvs, nil +} + +func EnsureZadigServiceByManifest(ctx context.Context, productName, namespace, manifest string) error { + env, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: productName, + Namespace: namespace, + }) + if err != nil { + return fmt.Errorf("failed to query namespace %q in project %q: %s", namespace, productName, err) + } + + if !env.ShareEnv.Enable { + return nil + } + + svcNames, err := util.GetSvcNamesFromManifest(manifest) + if err != nil { + return fmt.Errorf("failed to get Service names from manifest: %s", err) + } + + kclient, err := kubeclient.GetKubeClient(config.HubServerAddress(), env.ClusterID) + if err != nil { + return fmt.Errorf("failed to get kube client: %s", err) + } + + restConfig, err := kubeclient.GetRESTConfig(config.HubServerAddress(), env.ClusterID) + if err != nil { + return fmt.Errorf("failed to get rest config: %s", err) + } + + istioClient, err := versionedclient.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("failed to get istio client: %s", err) + } + + for _, svcName := range svcNames { + err := EnsureUpdateZadigService(ctx, env, svcName, kclient, istioClient) + if err != nil { + return fmt.Errorf("failed to ensure Zadig Service for K8s Service %q in env %q of product %q: %s", svcName, env.EnvName, env.ProductName, err) + } + } + + return nil +} + +func EnsureDeleteZadigServiceByHelmRelease(ctx context.Context, env *commonmodels.Product, releaseName string, helmClient helmclient.Client) error { + if !env.ShareEnv.Enable { + return nil + } + + release, err := helmClient.GetRelease(releaseName) + if err != nil { + return fmt.Errorf("failed to get release %q in namespace %q: %s", releaseName, env.Namespace, err) + } + + svcNames, err := util.GetSvcNamesFromManifest(release.Manifest) + if err != nil { + return fmt.Errorf("failed to get Service names from manifest: %s", err) + } + + kclient, err := kubeclient.GetKubeClient(config.HubServerAddress(), env.ClusterID) + if err != nil { + return fmt.Errorf("failed to get kube client: %s", err) + } + + restConfig, err := kubeclient.GetRESTConfig(config.HubServerAddress(), env.ClusterID) + if err != nil { + return fmt.Errorf("failed to get rest config: %s", err) + } + + istioClient, err := versionedclient.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("failed to get istio client: %s", err) + } + + for _, svcName := range svcNames { + err = ensureDeleteZadigServiceBySvcName(ctx, env, svcName, kclient, istioClient) + if err != nil { + return fmt.Errorf("failed to ensure deleting Zadig service by service name %q for env %q of product %q: %s", svcName, env.EnvName, env.ProductName, err) + } + } + + return nil +} + +func ensureDeleteZadigServiceBySvcName(ctx context.Context, env *commonmodels.Product, svcName string, kclient client.Client, istioClient versionedclient.Interface) error { + svc := &corev1.Service{} + err := kclient.Get(ctx, client.ObjectKey{ + Name: svcName, + Namespace: env.Namespace, + }, svc) + if err != nil { + return fmt.Errorf("failed to find Service %q in namespace %q: %s", svcName, env.Namespace, err) + } + + return ensureDeleteZadigService(ctx, env, svc, kclient, istioClient) +} + +func EnsureDeletePreCreatedServices(ctx context.Context, productName, namespace string, chartSpec *helmclient.ChartSpec, helmClient *helmtool.HelmClient) error { + env, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: productName, + Namespace: namespace, + }) + if err != nil { + return fmt.Errorf("failed to query namespace %q in project %q: %s", namespace, productName, err) + } + + if !(env.ShareEnv.Enable && !env.ShareEnv.IsBase) { + return nil + } + + manifestBytes, err := helmClient.TemplateChart(chartSpec) + if err != nil { + return fmt.Errorf("failed template chart %q for release %q in namespace %q: %s", chartSpec.ChartName, chartSpec.ReleaseName, chartSpec.Namespace, err) + } + + svcNames, err := util.GetSvcNamesFromManifest(string(manifestBytes)) + if err != nil { + return fmt.Errorf("failed to get Service names from manifest: %s", err) + } + + kclient, err := kubeclient.GetKubeClient(config.HubServerAddress(), env.ClusterID) + if err != nil { + return fmt.Errorf("failed to get kube client: %s", err) + } + + for _, svcName := range svcNames { + err := ensureDeleteK8sService(ctx, namespace, svcName, kclient, true) + if err != nil { + return fmt.Errorf("failed to ensure delete existing K8s Service %q in namespace %q: %s", svcName, namespace, err) + } + } + + return nil +} diff --git a/pkg/microservice/aslan/core/environment/service/share_env_utils.go b/pkg/microservice/aslan/core/environment/service/share_env_utils.go new file mode 100644 index 0000000000000000000000000000000000000000..b1616d26fc732bd3ab498d61a79e34deef0d8377 --- /dev/null +++ b/pkg/microservice/aslan/core/environment/service/share_env_utils.go @@ -0,0 +1,408 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "context" + "fmt" + + "go.uber.org/zap" + networkingv1alpha3 "istio.io/api/networking/v1alpha3" + versionedclient "istio.io/client-go/pkg/clientset/versioned" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/controller-runtime/pkg/client" + + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/tool/kube/util" + "github.com/koderover/zadig/pkg/tool/log" + zadiglabels "github.com/koderover/zadig/pkg/types" + zadigutil "github.com/koderover/zadig/pkg/util" +) + +func ensureBaseEnvConfig(ctx context.Context, baseEnv *commonmodels.Product) error { + if baseEnv.ShareEnv.Enable && baseEnv.ShareEnv.IsBase { + return nil + } + + baseEnv.ShareEnv = commonmodels.ProductShareEnv{ + Enable: true, + IsBase: true, + } + + return commonrepo.NewProductColl().Update(baseEnv) +} + +func ensureDisableBaseEnvConfig(ctx context.Context, baseEnv *commonmodels.Product) error { + if !baseEnv.ShareEnv.Enable && !baseEnv.ShareEnv.IsBase { + return nil + } + + baseEnv.ShareEnv = commonmodels.ProductShareEnv{ + Enable: false, + IsBase: false, + } + + return commonrepo.NewProductColl().Update(baseEnv) +} + +func ensureDeleteAssociatedEnvs(ctx context.Context, baseProduct *commonmodels.Product) error { + envs, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{ + Name: baseProduct.ProductName, + ShareEnvEnable: zadigutil.GetBoolPointer(true), + ShareEnvIsBase: zadigutil.GetBoolPointer(false), + ShareEnvBaseEnv: zadigutil.GetStrPointer(baseProduct.EnvName), + }) + if err != nil { + return err + } + + logger, err := zap.NewProduction() + if err != nil { + return err + } + + for _, env := range envs { + err := DeleteProduct("system", env.EnvName, env.ProductName, "", true, logger.Sugar()) + if err != nil { + return err + } + } + + return nil +} + +func ensureDeleteEnvoyFilter(ctx context.Context, baseEnv *commonmodels.Product, istioClient versionedclient.Interface) error { + envs, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{ + ShareEnvEnable: zadigutil.GetBoolPointer(true), + }) + if err != nil { + return fmt.Errorf("failed to list products which enable env sharing: %s", err) + } + + needDeleteEnvoyFilter := true + for _, env := range envs { + if env.ProductName == baseEnv.ProductName && env.EnvName == baseEnv.EnvName { + continue + } + + needDeleteEnvoyFilter = false + } + + if !needDeleteEnvoyFilter { + return nil + } + + return deleteEnvoyFilter(ctx, istioClient, istioNamespace, zadigEnvoyFilter) +} + +func ensureCleanRoutesInBase(ctx context.Context, grayEnv *commonmodels.Product, istioClient versionedclient.Interface) error { + baseEnv, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: grayEnv.ProductName, + EnvName: grayEnv.ShareEnv.BaseEnv, + }) + if err != nil { + return err + } + + grayNS := grayEnv.Namespace + vsList, err := istioClient.NetworkingV1alpha3().VirtualServices(grayNS).List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("failed to list VirtualServices for env %s of product %s: %s", grayEnv.EnvName, grayEnv.ProductName, err) + } + + for _, vsInGray := range vsList.Items { + err = ensureCleanRouteInBase(ctx, grayEnv.EnvName, baseEnv.Namespace, vsInGray.Name, istioClient) + if err != nil { + return fmt.Errorf("failed to clean route in base env: %s", err) + } + } + + return nil +} + +func deleteEnvoyFilter(ctx context.Context, istioClient versionedclient.Interface, istioNamespace, name string) error { + _, err := istioClient.NetworkingV1alpha3().EnvoyFilters(istioNamespace).Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + log.Infof("EnvoyFilter %s is not found in ns `%s`. Skip.", name, istioNamespace) + return nil + } + if err != nil { + return fmt.Errorf("failed to query EnvoyFilter %s in ns `%s`: %s", name, istioNamespace, err) + } + + deleteOption := metav1.DeletePropagationBackground + return istioClient.NetworkingV1alpha3().EnvoyFilters(istioNamespace).Delete(ctx, name, metav1.DeleteOptions{ + PropagationPolicy: &deleteOption, + }) +} + +func ensureCleanRouteInBase(ctx context.Context, envName, baseNS, vsName string, istioClient versionedclient.Interface) error { + vsObj, err := istioClient.NetworkingV1alpha3().VirtualServices(baseNS).Get(ctx, vsName, metav1.GetOptions{}) + if err != nil { + log.Warnf("Failed to query VirtualService %s releated to env %s in namesapce %s: %s. It may be not an exception and skip.", vsName, envName, baseNS, err) + return nil + } + + // Note: DeepCopy is used to avoid unpredictable results in concurrent operations. + baseVS := vsObj.DeepCopy() + + if len(baseVS.Spec.Http) == 0 { + return nil + } + + var needClean bool + var location int + for i, vsHttp := range baseVS.Spec.Http { + if len(vsHttp.Match) == 0 { + continue + } + + for _, vsMatch := range vsHttp.Match { + if len(vsMatch.Headers) == 0 { + continue + } + + matchValue, found := vsMatch.Headers[zadigMatchXEnv] + if !found { + continue + } + + if matchValue.GetExact() != envName { + continue + } + + needClean = true + break + } + + if !needClean { + continue + } + + location = i + break + } + + if !needClean { + return nil + } + + log.Infof("Begin to clean route env=%s in base ns %s for VirtualService %s.", envName, baseNS, vsName) + + httpRoutes := make([]*networkingv1alpha3.HTTPRoute, 0, len(baseVS.Spec.Http)-1) + httpRoutes = append(httpRoutes, baseVS.Spec.Http[:location]...) + httpRoutes = append(httpRoutes, baseVS.Spec.Http[location+1:]...) + baseVS.Spec.Http = httpRoutes + + _, err = istioClient.NetworkingV1alpha3().VirtualServices(baseNS).Update(ctx, baseVS, metav1.UpdateOptions{}) + return err +} + +func ensureServicesInAllSubEnvs(ctx context.Context, env *commonmodels.Product, svc *corev1.Service, kclient client.Client, istioClient versionedclient.Interface) error { + envs, err := fetchSubEnvs(ctx, env.ProductName, env.ClusterID, env.EnvName) + if err != nil { + return err + } + + vsName := genVirtualServiceName(svc) + for _, env := range envs { + log.Infof("Begin to ensure Services in subenv %s of prouduct %s.", env.EnvName, env.ProductName) + + err = ensureVirtualService(ctx, kclient, istioClient, env, svc, vsName) + if err != nil { + return err + } + + err = ensureDefaultK8sServiceInGray(ctx, svc, env.Namespace, kclient) + if err != nil { + return err + } + } + + return nil +} + +func ensureDeleteVirtualService(ctx context.Context, env *commonmodels.Product, vsName string, istioClient versionedclient.Interface) error { + _, err := istioClient.NetworkingV1alpha3().VirtualServices(env.Namespace).Get(ctx, vsName, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + log.Warnf("Failed to find VirtualService %s for env %s of product %s. Don't try to delete it.", vsName, env.EnvName, env.ProductName) + return nil + } + if err != nil { + return err + } + + deleteOption := metav1.DeletePropagationBackground + return istioClient.NetworkingV1alpha3().VirtualServices(env.Namespace).Delete(ctx, vsName, metav1.DeleteOptions{ + PropagationPolicy: &deleteOption, + }) +} + +func ensureDeleteServiceInAllSubEnvs(ctx context.Context, baseEnv *commonmodels.Product, svc *corev1.Service, kclient client.Client, istioClient versionedclient.Interface) error { + envs, err := fetchSubEnvs(ctx, baseEnv.ProductName, baseEnv.ClusterID, baseEnv.EnvName) + if err != nil { + return err + } + + // Note: Don't delete VirtualService and K8s Service if there's selected pods. + vsName := genVirtualServiceName(svc) + workloadSelector := labels.SelectorFromSet(labels.Set(svc.Spec.Selector)) + for _, env := range envs { + hasWorkload, err := doesSvcHasWorkload(ctx, env.Namespace, workloadSelector, kclient) + if err != nil { + return err + } + if hasWorkload { + continue + } + + err = ensureDeleteVirtualService(ctx, env, vsName, istioClient) + if err != nil { + return fmt.Errorf("failed to delete VirtualService %s in env %s of product %s: %s", vsName, env.EnvName, env.ProductName, err) + } + + err = ensureDeleteK8sService(ctx, env.Namespace, svc.Name, kclient, true) + if err != nil { + return fmt.Errorf("failed to delete K8s Service %s in env %s of product: %s: %s", svc.Name, env.EnvName, env.ProductName, err) + } + } + + return nil +} + +func ensureDeleteK8sService(ctx context.Context, ns, svcName string, kclient client.Client, systemCreatedOnly bool) error { + svc := &corev1.Service{} + err := kclient.Get(ctx, client.ObjectKey{ + Name: svcName, + Namespace: ns, + }, svc) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return err + } + + if systemCreatedOnly && !(svc.Labels != nil && svc.Labels[zadiglabels.ZadigLabelKeyGlobalOwner] == zadiglabels.Zadig) { + return nil + } + + deleteOption := metav1.DeletePropagationBackground + return kclient.Delete(ctx, svc, &client.DeleteOptions{ + PropagationPolicy: &deleteOption, + }) +} + +func doesSvcHasWorkload(ctx context.Context, ns string, svcSelector labels.Selector, kclient client.Client) (bool, error) { + podList := &corev1.PodList{} + err := kclient.List(ctx, podList, &client.ListOptions{ + Namespace: ns, + LabelSelector: svcSelector, + }) + if err != nil { + return false, util.IgnoreNotFoundError(err) + } + + if len(podList.Items) == 0 { + return false, nil + } + + return true, nil +} + +func fetchSubEnvs(ctx context.Context, productName, clusterID, baseEnvName string) ([]*commonmodels.Product, error) { + return commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{ + Name: productName, + ClusterID: clusterID, + ShareEnvEnable: zadigutil.GetBoolPointer(true), + ShareEnvIsBase: zadigutil.GetBoolPointer(false), + ShareEnvBaseEnv: zadigutil.GetStrPointer(baseEnvName), + }) +} + +func getSvcInEnv(env *commonmodels.Product) []string { + svcs := []string{} + for _, svcGroup := range env.Services { + for _, svc := range svcGroup { + svcs = append(svcs, svc.ServiceName) + } + } + + return svcs +} + +func ensureUpdateZadigSerivce(ctx context.Context, env *commonmodels.Product, svc *corev1.Service, kclient client.Client, istioClient versionedclient.Interface) error { + vsName := genVirtualServiceName(svc) + + if env.ShareEnv.IsBase { + // 1. Create VirtualService in the base environment. + err := ensureVirtualService(ctx, kclient, istioClient, env, svc, vsName) + if err != nil { + return err + } + + // 2. Create VirtualService in all of the sub environments. + return ensureServicesInAllSubEnvs(ctx, env, svc, kclient, istioClient) + } + + baseEnv, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: env.ProductName, + EnvName: env.ShareEnv.BaseEnv, + }) + if err != nil { + return err + } + + // 1. Create VirtualService in the sub environment. + err = ensureVirtualServiceInGray(ctx, env.EnvName, vsName, svc.Name, env.Namespace, baseEnv.Namespace, istioClient) + if err != nil { + return err + } + + // 2. Updated the VirtualService configuration in the base environment. + return ensureUpdateVirtualServiceInBase(ctx, env.EnvName, vsName, svc.Name, env.Namespace, baseEnv.Namespace, istioClient) +} + +func ensureDeleteZadigService(ctx context.Context, env *commonmodels.Product, svc *corev1.Service, kclient client.Client, istioClient versionedclient.Interface) error { + vsName := genVirtualServiceName(svc) + + // Delete VirtualService in the current environment. + err := ensureDeleteVirtualService(ctx, env, vsName, istioClient) + if err != nil { + return err + } + + if env.ShareEnv.IsBase { + // Delete VirtualService and K8s Service in all of the sub environments if there're no specific workloads. + return ensureDeleteServiceInAllSubEnvs(ctx, env, svc, kclient, istioClient) + } + + baseEnv, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: env.ProductName, + EnvName: env.ShareEnv.BaseEnv, + }) + if err != nil { + return err + } + + // Update VirtualService Routes in the base environment. + return ensureCleanRouteInBase(ctx, env.EnvName, baseEnv.Namespace, vsName, istioClient) +} diff --git a/pkg/microservice/aslan/core/environment/service/types.go b/pkg/microservice/aslan/core/environment/service/types.go index 60942bf80bce957cee213812eec0f89f41914fc3..48d0b3d10de4772bb1228d80d105515adec5a797 100644 --- a/pkg/microservice/aslan/core/environment/service/types.go +++ b/pkg/microservice/aslan/core/environment/service/types.go @@ -115,3 +115,83 @@ type ContainerNotFound struct { func (c *ContainerNotFound) Error() string { return fmt.Sprintf("serviceName:%s,container:%s", c.ServiceName, c.Container) } + +type NodeResp struct { + Nodes []*internalresource.Node `json:"data"` + Labels []string `json:"labels"` +} + +type ShareEnvReady struct { + IsReady bool `json:"is_ready"` + Checks ShareEnvReadyChecks `json:"checks"` +} + +type ShareEnvReadyChecks struct { + NamespaceHasIstioLabel bool `json:"namespace_has_istio_label"` + VirtualServicesDeployed bool `json:"virtualservice_deployed"` + PodsHaveIstioProxy bool `json:"pods_have_istio_proxy"` + WorkloadsReady bool `json:"workloads_ready"` + WorkloadsHaveK8sService bool `json:"workloads_have_k8s_service"` +} + +// Note: `WorkloadsHaveK8sService` is an optional condition. +func (s *ShareEnvReady) CheckAndSetReady(state ShareEnvOp) { + if !s.Checks.WorkloadsReady { + s.IsReady = false + return + } + + switch state { + case ShareEnvEnable: + if !s.Checks.NamespaceHasIstioLabel || !s.Checks.VirtualServicesDeployed || !s.Checks.PodsHaveIstioProxy { + s.IsReady = false + } else { + s.IsReady = true + } + default: + if !s.Checks.NamespaceHasIstioLabel && !s.Checks.VirtualServicesDeployed && !s.Checks.PodsHaveIstioProxy { + s.IsReady = true + } else { + s.IsReady = false + } + } +} + +type EnvoyClusterConfigLoadAssignment struct { + ClusterName string `json:"cluster_name"` + Endpoints []EnvoyLBEndpoints `json:"endpoints"` +} + +type EnvoyLBEndpoints struct { + LBEndpoints []EnvoyEndpoints `json:"lb_endpoints"` +} + +type EnvoyEndpoints struct { + Endpoint EnvoyEndpoint `json:"endpoint"` +} + +type EnvoyEndpoint struct { + Address EnvoyAddress `json:"address"` +} + +type EnvoyAddress struct { + SocketAddress EnvoySocketAddress `json:"socket_address"` +} + +type EnvoySocketAddress struct { + Protocol string `json:"protocol"` + Address string `json:"address"` + PortValue int `json:"port_value"` +} + +type ShareEnvOp string + +const ( + ShareEnvEnable ShareEnvOp = "enable" + ShareEnvDisable ShareEnvOp = "disable" +) + +type MatchedEnv struct { + EnvName string + Namespace string +} diff --git a/pkg/microservice/aslan/core/label/config/consts.go b/pkg/microservice/aslan/core/label/config/consts.go new file mode 100644 index 0000000000000000000000000000000000000000..744761b172b4ee2773233a370235901547754860 --- /dev/null +++ b/pkg/microservice/aslan/core/label/config/consts.go @@ -0,0 +1,24 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +type ResourceType string + +const ( + ResourceTypeWorkflow ResourceType = "Workflow" + ResourceTypeEnvironment ResourceType = "Environment" +) diff --git a/pkg/microservice/aslan/core/label/handler/label_binding.go b/pkg/microservice/aslan/core/label/handler/label_binding.go new file mode 100644 index 0000000000000000000000000000000000000000..a699b73b4840f5578479c51f6eb118f2aeb6a76f --- /dev/null +++ b/pkg/microservice/aslan/core/label/handler/label_binding.go @@ -0,0 +1,68 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/label/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/label/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +func lBValidate(lb []*mongodb.LabelBinding) error { + for _, v := range lb { + if v.LabelID == "" || v.Resource.Type == "" { + return e.ErrInvalidParam.AddDesc("invalid labelbinding args") + } + } + return nil +} + +func CreateLabelBindings(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + createLabelBindingsArgs := new(service.CreateLabelBindingsArgs) + if err := c.ShouldBindJSON(createLabelBindingsArgs); err != nil { + ctx.Err = err + return + } + if err := lBValidate(createLabelBindingsArgs.LabelBindings); err != nil { + ctx.Err = err + return + } + ctx.Err = service.CreateLabelBindings(createLabelBindingsArgs, ctx.UserName, ctx.Logger) +} + +func DeleteLabelBindings(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + deleteLabelBindingsArgs := new(service.DeleteLabelBindingsArgs) + if err := c.ShouldBindJSON(deleteLabelBindingsArgs); err != nil { + ctx.Err = err + return + } + if err := lBValidate(deleteLabelBindingsArgs.LabelBindings); err != nil { + ctx.Err = err + return + } + ctx.Logger.Infof("DeleteLabelBindings user:%s", ctx.UserName) + ctx.Err = service.DeleteLabelBindings(deleteLabelBindingsArgs, ctx.UserName, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/label/handler/labels.go b/pkg/microservice/aslan/core/label/handler/labels.go new file mode 100644 index 0000000000000000000000000000000000000000..b8274818db1fcf9854d3ee4d0dfe6d867c237364 --- /dev/null +++ b/pkg/microservice/aslan/core/label/handler/labels.go @@ -0,0 +1,107 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "strconv" + + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/label/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +func ListLabels(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + listLabelsArgs := new(service.ListLabelsArgs) + if err := c.ShouldBindJSON(&listLabelsArgs); err != nil { + ctx.Err = err + return + } + ctx.Resp, ctx.Err = service.ListLabels(listLabelsArgs) +} + +func createLabelValidate(lbs *service.CreateLabelsArgs) error { + for _, v := range lbs.Labels { + if v.Key == "" || v.Value == "" { + return e.ErrInvalidParam.AddDesc("invalid label args") + } + } + return nil +} + +func CreateLabels(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + labels := new(service.CreateLabelsArgs) + if err := c.ShouldBindJSON(&labels); err != nil { + ctx.Err = err + return + } + + if err := createLabelValidate(labels); err != nil { + ctx.Err = err + return + } + + ctx.Resp, ctx.Err = service.CreateLabels(labels, ctx.UserName) +} + +// DeleteLabels can only bulk delete labels which do not bind resources except add force param +func DeleteLabels(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + deleteLabelsArgs := new(service.DeleteLabelsArgs) + if err := c.ShouldBindJSON(deleteLabelsArgs); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("json bind fail") + return + } + force := c.Query("force") + forceBool, err := strconv.ParseBool(force) + if err != nil { + ctx.Err = err + return + } + ctx.Err = service.DeleteLabels(deleteLabelsArgs.IDs, forceBool, ctx.UserName, ctx.Logger) +} + +func ListResourcesByLabels(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + listResourceByLabelsReq := new(service.ListResourceByLabelsReq) + if err := c.ShouldBindJSON(listResourceByLabelsReq); err != nil { + ctx.Err = err + return + } + ctx.Resp, ctx.Err = service.ListResourcesByLabels(listResourceByLabelsReq.LabelFilters, ctx.Logger) +} + +func ListLabelsByResources(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + listLabelsByResourcesReq := new(service.ListLabelsByResourcesReq) + + if err := c.ShouldBindJSON(&listLabelsByResourcesReq); err != nil { + ctx.Err = err + return + } + ctx.Resp, ctx.Err = service.ListLabelsByResources(listLabelsByResourcesReq.Resources, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/label/handler/router.go b/pkg/microservice/aslan/core/label/handler/router.go new file mode 100644 index 0000000000000000000000000000000000000000..83328ea5bf2bcb92d4ea8436a9ed2feac1f568f2 --- /dev/null +++ b/pkg/microservice/aslan/core/label/handler/router.go @@ -0,0 +1,39 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" +) + +type Router struct{} + +func (*Router) Inject(router *gin.RouterGroup) { + labels := router.Group("labels") + { + labels.POST("/filter", ListLabels) + labels.POST("/bulk-create", CreateLabels) + labels.POST("/bulk-delete", DeleteLabels) + labels.POST("/resources-by-labels", ListResourcesByLabels) + labels.POST("/labels-by-resources", ListLabelsByResources) + } + labelBindings := router.Group("labelbindings") + { + labelBindings.POST("", CreateLabelBindings) + labelBindings.DELETE("", DeleteLabelBindings) + } +} diff --git a/pkg/microservice/aslan/core/label/repository/models/label.go b/pkg/microservice/aslan/core/label/repository/models/label.go new file mode 100644 index 0000000000000000000000000000000000000000..ef9c0f920691a2f80bdc53ef9339cc379ff4ef1b --- /dev/null +++ b/pkg/microservice/aslan/core/label/repository/models/label.go @@ -0,0 +1,37 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" + + "github.com/koderover/zadig/pkg/setting" +) + +type Label struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + Type setting.ResourceType `bson:"type" json:"type"` + Key string `bson:"key" json:"key"` + Value string `bson:"value" json:"value"` + CreateBy string `bson:"create_by" json:"create_by"` + CreateTime int64 `bson:"create_time" json:"create_time"` + ProjectName string `bson:"project_name" json:"project_name"` +} + +func (Label) TableName() string { + return "label" +} diff --git a/pkg/microservice/aslan/core/label/repository/models/labelbinding.go b/pkg/microservice/aslan/core/label/repository/models/labelbinding.go new file mode 100644 index 0000000000000000000000000000000000000000..e67f86e614ef7882422aa34051c56cb96cee3ebb --- /dev/null +++ b/pkg/microservice/aslan/core/label/repository/models/labelbinding.go @@ -0,0 +1,35 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type LabelBinding struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + ResourceType string `bson:"resource_type" json:"resource_type"` + ResourceName string `bson:"resource_name" json:"resource_name"` + ProjectName string `bson:"project_name" json:"project_name"` + LabelID string `bson:"label_id" json:"label_id"` + CreateBy string `bson:"create_by" json:"create_by"` + CreateTime int64 `bson:"create_time" json:"create_time"` +} + +func (LabelBinding) TableName() string { + return "labelbinding" +} diff --git a/pkg/microservice/aslan/core/label/repository/mongodb/label.go b/pkg/microservice/aslan/core/label/repository/mongodb/label.go new file mode 100644 index 0000000000000000000000000000000000000000..364025b8cab6ab6ad6b9998ef982d7c53895541a --- /dev/null +++ b/pkg/microservice/aslan/core/label/repository/mongodb/label.go @@ -0,0 +1,207 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/label/repository/models" + "github.com/koderover/zadig/pkg/setting" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type LabelColl struct { + *mongo.Collection + coll string +} + +func NewLabelColl() *LabelColl { + name := models.Label{}.TableName() + return &LabelColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} +} + +func (c *LabelColl) GetCollectionName() string { + return c.coll +} + +func (c *LabelColl) EnsureIndex(ctx context.Context) error { + mod := mongo.IndexModel{ + Keys: bson.D{ + bson.E{Key: "key", Value: 1}, + bson.E{Key: "value", Value: 1}, + }, + Options: options.Index().SetUnique(true), + } + _, err := c.Indexes().CreateOne(ctx, mod) + return err +} + +func (c *LabelColl) BulkCreate(labels []*models.Label) error { + if len(labels) == 0 { + return nil + } + var ois []interface{} + for _, label := range labels { + label.CreateTime = time.Now().Unix() + ois = append(ois, label) + } + + _, err := c.InsertMany(context.TODO(), ois) + return err +} + +func (c *LabelColl) Delete(id string) error { + oid, err := primitive.ObjectIDFromHex(id) + if err != nil { + return err + } + query := bson.M{"_id": oid} + _, err = c.DeleteOne(context.TODO(), query) + return err +} + +func (c *LabelColl) BulkDelete(ids []string) error { + if len(ids) == 0 { + return nil + } + var oids []primitive.ObjectID + for _, v := range ids { + oid, err := primitive.ObjectIDFromHex(v) + if err != nil { + return err + } + oids = append(oids, oid) + } + query := bson.M{} + query["_id"] = bson.M{"$in": oids} + _, err := c.DeleteMany(context.TODO(), query) + return err +} + +func (c *LabelColl) BulkDeleteByProject(projectName string) error { + if projectName == "" { + return nil + } + + query := bson.M{} + query["project_name"] = projectName + _, err := c.DeleteMany(context.TODO(), query) + return err +} + +func (c *LabelColl) ListByProjectName(projectName string) ([]*models.Label, error) { + if len(projectName) == 0 { + return nil, nil + } + query := bson.M{} + var res []*models.Label + query["project_name"] = projectName + ctx := context.Background() + cursor, err := c.Collection.Find(ctx, query) + if err != nil { + return nil, err + } + err = cursor.All(ctx, &res) + if err != nil { + return nil, err + } + return res, nil +} + +func (c *LabelColl) Find(id string) (*models.Label, error) { + res := new(models.Label) + oid, err := primitive.ObjectIDFromHex(id) + if err != nil { + return nil, err + } + opts := options.FindOne() + query := bson.M{"_id": oid} + err = c.FindOne(context.TODO(), query, opts).Decode(res) + return res, err +} + +type Label struct { + Key string `json:"key"` + Value string `json:"value"` + Type setting.ResourceType `json:"type"` + ProjectName string `json:"project_name"` +} + +type ListLabelOpt struct { + Labels []Label +} + +func (c *LabelColl) List(opt ListLabelOpt) ([]*models.Label, error) { + var res []*models.Label + if len(opt.Labels) == 0 { + return nil, nil + } + condition := bson.A{} + for _, label := range opt.Labels { + condition = append(condition, bson.M{ + "key": label.Key, + "value": label.Value, + }) + } + filter := bson.D{{"$or", condition}} + cursor, err := c.Collection.Find(context.TODO(), filter) + if err == mongo.ErrNoDocuments { + return nil, nil + } + if err != nil { + return nil, err + } + if err := cursor.All(context.TODO(), &res); err != nil { + return nil, err + } + return res, nil +} + +func (c *LabelColl) ListByIDs(ids []string) ([]*models.Label, error) { + var res []*models.Label + ctx := context.Background() + query := bson.M{} + if len(ids) != 0 { + var oids []primitive.ObjectID + for _, id := range ids { + oid, err := primitive.ObjectIDFromHex(id) + if err != nil { + return nil, err + } + oids = append(oids, oid) + } + query["_id"] = bson.M{"$in": oids} + } + opts := options.Find() + opts.SetSort(bson.D{{"key", 1}}) + cursor, err := c.Collection.Find(ctx, query) + if err != nil { + return nil, err + } + err = cursor.All(ctx, &res) + if err != nil { + return nil, err + } + return res, nil +} diff --git a/pkg/microservice/aslan/core/label/repository/mongodb/labelbinding.go b/pkg/microservice/aslan/core/label/repository/mongodb/labelbinding.go new file mode 100644 index 0000000000000000000000000000000000000000..4be91140100e1a9158f3e81de7ed50204690405c --- /dev/null +++ b/pkg/microservice/aslan/core/label/repository/mongodb/labelbinding.go @@ -0,0 +1,246 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/label/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type LabelBindingColl struct { + *mongo.Collection + + coll string +} + +func NewLabelBindingColl() *LabelBindingColl { + name := models.LabelBinding{}.TableName() + return &LabelBindingColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} +} + +func (c *LabelBindingColl) GetCollectionName() string { + return c.coll +} + +func (c *LabelBindingColl) EnsureIndex(ctx context.Context) error { + mod := mongo.IndexModel{ + Keys: bson.D{ + bson.E{Key: "resource_name", Value: 1}, + bson.E{Key: "project_name", Value: 1}, + bson.E{Key: "label_id", Value: 1}, + bson.E{Key: "resource_type", Value: 1}, + }, + Options: options.Index().SetUnique(true), + } + + _, err := c.Indexes().CreateOne(ctx, mod) + return err +} + +type LabelBinding struct { + Resource Resource `json:"resource" bson:"resource"` + LabelID string `json:"label_id" bson:"label_id"` + CreateBy string `json:"create_by" bson:"create_by"` + CreateTime int64 `json:"create_time" bson:"create_time"` +} + +func (c *LabelBindingColl) CreateMany(labelBindings []*LabelBinding) error { + if len(labelBindings) == 0 { + return nil + } + var lbs []models.LabelBinding + for _, labelBinding := range labelBindings { + tmplb := models.LabelBinding{ + ResourceType: labelBinding.Resource.Type, + ResourceName: labelBinding.Resource.Name, + ProjectName: labelBinding.Resource.ProjectName, + LabelID: labelBinding.LabelID, + CreateBy: labelBinding.CreateBy, + CreateTime: time.Now().Unix(), + } + lbs = append(lbs, tmplb) + } + + var ois []interface{} + for _, obj := range lbs { + ois = append(ois, obj) + } + _, err := c.InsertMany(context.TODO(), ois) + return err +} + +type LabelBindingCollFindOpt struct { + LabelID string + LabelIDs []string + ResourceID string + ResourcesIDs []string + ResourceType string +} + +func (c *LabelBindingColl) FindByOpt(opt *LabelBindingCollFindOpt) (*models.LabelBinding, error) { + res := &models.LabelBinding{} + query := bson.M{} + if opt.LabelID != "" { + query["label_id"] = opt.LabelID + } + err := c.FindOne(context.TODO(), query).Decode(res) + return res, err +} + +func (c *LabelBindingColl) ListByOpt(opt *LabelBindingCollFindOpt) ([]*models.LabelBinding, error) { + var ret []*models.LabelBinding + query := bson.M{} + if opt.LabelID != "" { + query["label_id"] = opt.LabelID + } + if len(opt.LabelIDs) != 0 { + query["label_id"] = bson.M{"$in": opt.LabelIDs} + } + if opt.ResourceID != "" { + query["resource_id"] = opt.ResourceID + } + if len(opt.ResourcesIDs) != 0 { + query["resource_id"] = bson.M{"$in": opt.ResourcesIDs} + } + if opt.ResourceType != "" { + query["resource_type"] = opt.ResourceType + } + ctx := context.Background() + + cursor, err := c.Collection.Find(ctx, query) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &ret) + if err != nil { + return nil, err + } + return ret, err +} + +func (c *LabelBindingColl) BulkDeleteByProject(projectName string) error { + if projectName == "" { + return nil + } + + query := bson.M{} + query["project_name"] = projectName + _, err := c.DeleteMany(context.TODO(), query) + return err +} + +func (c *LabelBindingColl) BulkDeleteByLabelIds(labelIds []string) error { + if len(labelIds) == 0 { + return nil + } + + condition := bson.A{} + for _, id := range labelIds { + condition = append(condition, bson.M{ + "label_id": id, + }) + } + query := bson.D{{"$or", condition}} + _, err := c.DeleteMany(context.TODO(), query) + return err +} + +func (c *LabelBindingColl) BulkDeleteByIds(ids []string) error { + if len(ids) == 0 { + return nil + } + + condition := bson.A{} + for _, id := range ids { + oid, err := primitive.ObjectIDFromHex(id) + if err != nil { + return err + } + condition = append(condition, bson.M{ + "_id": oid, + }) + } + query := bson.D{{"$or", condition}} + _, err := c.DeleteMany(context.TODO(), query) + return err +} + +func (c *LabelBindingColl) BulkDelete(labelBindings []*LabelBinding) error { + if len(labelBindings) == 0 { + return nil + } + condition := bson.A{} + for _, binding := range labelBindings { + condition = append(condition, bson.M{ + "label_id": binding.LabelID, + "resource_name": binding.Resource.Name, + "project_name": binding.Resource.ProjectName, + "resource_type": binding.Resource.Type, + }) + } + query := bson.D{{"$or", condition}} + _, err := c.DeleteMany(context.TODO(), query) + return err +} + +type Resource struct { + Name string `json:"name"` + ProjectName string `json:"project_name"` + Type string `json:"type"` +} + +type ListLabelBindingsByResources struct { + Resources []Resource +} + +func (c *LabelBindingColl) ListByResources(opt ListLabelBindingsByResources) ([]*models.LabelBinding, error) { + var res []*models.LabelBinding + + if len(opt.Resources) == 0 { + return nil, nil + } + condition := bson.A{} + for _, resource := range opt.Resources { + condition = append(condition, bson.M{ + "resource_type": resource.Type, + "resource_name": resource.Name, + "project_name": resource.ProjectName, + }) + } + filter := bson.D{{"$or", condition}} + cursor, err := c.Collection.Find(context.TODO(), filter) + if err == mongo.ErrNoDocuments { + return nil, nil + } + if err != nil { + return nil, err + } + if err := cursor.All(context.TODO(), &res); err != nil { + return nil, err + } + return res, nil +} diff --git a/pkg/microservice/aslan/core/label/service/label.go b/pkg/microservice/aslan/core/label/service/label.go new file mode 100644 index 0000000000000000000000000000000000000000..b2af9dda5870326dd213862c95fd42d54c72aa0e --- /dev/null +++ b/pkg/microservice/aslan/core/label/service/label.go @@ -0,0 +1,230 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "fmt" + + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/label/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/label/repository/mongodb" +) + +type CreateLabelsResp struct { + LabelMap map[string]string `json:"label_map"` +} + +func CreateLabels(arg *CreateLabelsArgs, userName string) (*CreateLabelsResp, error) { + filteredLabels := make([]*models.Label, 0) + keyValues := sets.NewString() + for _, v := range arg.Labels { + keyValue := fmt.Sprintf("%s-%s", v.Key, v.Value) + if keyValues.Has(keyValue) { + continue + } + keyValues.Insert(keyValue) + tmpModel := models.Label{ + Key: v.Key, + Value: v.Value, + CreateBy: userName, + Type: v.Type, + ProjectName: v.ProjectName, + } + filteredLabels = append(filteredLabels, &tmpModel) + } + err := mongodb.NewLabelColl().BulkCreate(filteredLabels) + if err != nil { + return nil, err + } + var labelModels []mongodb.Label + for _, label := range filteredLabels { + labelModels = append(labelModels, mongodb.Label{ + Key: label.Key, + Value: label.Value, + }) + } + result := make(map[string]string) + resp, err := mongodb.NewLabelColl().List(mongodb.ListLabelOpt{ + Labels: labelModels, + }) + for _, label := range resp { + result[BuildLabelString(label.Key, label.Value)] = label.ID.Hex() + } + return &CreateLabelsResp{ + LabelMap: result, + }, err +} + +type ListLabelsArgs struct { + Labels []mongodb.Label `json:"labels"` +} + +type ListLabelsResp struct { + Labels []*models.Label `json:"labels"` +} + +type CreateLabelsArgs struct { + Labels []mongodb.Label `json:"labels"` +} + +func ListLabels(args *ListLabelsArgs) (*ListLabelsResp, error) { + labels, err := mongodb.NewLabelColl().List(mongodb.ListLabelOpt{args.Labels}) + if err != nil { + return nil, err + } + return &ListLabelsResp{Labels: labels}, nil +} + +type ListResourceByLabelsReq struct { + LabelFilters []mongodb.Label `json:"label_filters"` +} + +type ListResourcesByLabelsResp struct { + Resources map[string][]mongodb.Resource `json:"resources"` +} + +func BuildLabelString(key string, value string) string { + return fmt.Sprintf("%s:%s", key, value) +} + +func ListResourcesByLabels(filters []mongodb.Label, logger *zap.SugaredLogger) (*ListResourcesByLabelsResp, error) { + res := make(map[string][]mongodb.Resource) + if len(filters) == 0 { + return &ListResourcesByLabelsResp{Resources: res}, nil + } + // 1.find labels by label filters + labels, err := mongodb.NewLabelColl().List(mongodb.ListLabelOpt{Labels: filters}) + if err != nil { + logger.Errorf("find labels by label filters err:%s", err) + return nil, err + } + if len(labels) == 0 { + return &ListResourcesByLabelsResp{Resources: res}, nil + } + // 2.find labelBindings by label ids + labelIDSet := sets.NewString() + labelsM := make(map[string]string) + for _, v := range labels { + labelIDSet.Insert(v.ID.Hex()) + labelsM[v.ID.Hex()] = BuildLabelString(v.Key, v.Value) + } + + labelBindings, err := mongodb.NewLabelBindingColl().ListByOpt(&mongodb.LabelBindingCollFindOpt{LabelIDs: labelIDSet.List()}) + if err != nil { + logger.Errorf("list labelBindings err:%s,ids:%v", err, labelIDSet.List()) + return nil, err + } + + // 3.find labels by resourceName-projectName + for _, v := range labelBindings { + resource := mongodb.Resource{ + Name: v.ResourceName, + ProjectName: v.ProjectName, + Type: v.ResourceType, + } + labelString, _ := labelsM[v.LabelID] + res[labelString] = append(res[labelString], resource) + } + + return &ListResourcesByLabelsResp{ + Resources: res, + }, nil +} + +type ListLabelsByResourcesReq struct { + Resources []mongodb.Resource `json:"resources"` +} + +type ListLabelsByResourcesResp struct { + Labels map[string][]*models.Label `json:"labels"` +} + +func ListLabelsByResources(resources []mongodb.Resource, logger *zap.SugaredLogger) (*ListLabelsByResourcesResp, error) { + //1. find the labelBindings by resources + labelBindings, err := mongodb.NewLabelBindingColl().ListByResources(mongodb.ListLabelBindingsByResources{Resources: resources}) + if err != nil { + logger.Errorf("find the labelBindings by resources err:%s", err) + return nil, err + } + + //2.find labels by labelBindings + labelIDSet := sets.NewString() + for _, v := range labelBindings { + labelIDSet.Insert(v.LabelID) + } + labels, err := mongodb.NewLabelColl().ListByIDs(labelIDSet.List()) + if err != nil { + return nil, err + } + labelM := make(map[string]*models.Label) + for _, label := range labels { + labelM[label.ID.Hex()] = label + } + // 3. iterate resources + res := make(map[string][]*models.Label) + for _, labelBinding := range labelBindings { + resourceKey := config.BuildResourceKey(labelBinding.ResourceType, labelBinding.ProjectName, labelBinding.ResourceName) + label, ok := labelM[labelBinding.LabelID] + if !ok { + logger.Errorf("can not find label %v", labelBinding) + continue + } + + res[resourceKey] = append(res[resourceKey], label) + + } + + return &ListLabelsByResourcesResp{ + Labels: res, + }, nil +} + +type DeleteLabelsArgs struct { + IDs []string +} + +func DeleteLabels(ids []string, forceDelete bool, userName string, logger *zap.SugaredLogger) error { + if len(ids) == 0 { + return nil + } + + if forceDelete { + logger.Infof("user:%s force deleteLabels:%s", userName, ids) + if err := mongodb.NewLabelBindingColl().BulkDeleteByLabelIds(ids); err != nil { + logger.Errorf("BulkDeleteByIds err :%s,ids:%v", err, ids) + return err + } + } + + return mongodb.NewLabelColl().BulkDelete(ids) +} + +func DeleteLabelsAndBindingsByProject(projectName string, logger *zap.SugaredLogger) error { + if err := mongodb.NewLabelBindingColl().BulkDeleteByProject(projectName); err != nil { + logger.Errorf("DeleteLabelsByProject err:%s", err) + return err + } + + if err := mongodb.NewLabelColl().BulkDeleteByProject(projectName); err != nil { + logger.Errorf("DeleteLabelsByProject err:%s", err) + return err + } + return nil +} diff --git a/pkg/microservice/aslan/core/label/service/labelsbinding.go b/pkg/microservice/aslan/core/label/service/labelsbinding.go new file mode 100644 index 0000000000000000000000000000000000000000..faf8c16167e2657f2c000740106b5cdd8351b6e9 --- /dev/null +++ b/pkg/microservice/aslan/core/label/service/labelsbinding.go @@ -0,0 +1,120 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "fmt" + + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/util/sets" + + commondb "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/label/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/label/repository/mongodb" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +type CreateLabelBindingsArgs struct { + LabelBindings []*mongodb.LabelBinding `json:"label_bindings"` +} + +func CreateLabelBindings(cr *CreateLabelBindingsArgs, userName string, logger *zap.SugaredLogger) error { + // check label exist + labelIDs := sets.String{} + for _, binding := range cr.LabelBindings { + labelIDs.Insert(binding.LabelID) + binding.CreateBy = userName + } + + labels, err := mongodb.NewLabelColl().ListByIDs(labelIDs.List()) + if err != nil { + logger.Errorf("find labels err:%s", err) + return err + } + if len(labels) != len(labelIDs) { + logger.Errorf("there're labels not exist") + return fmt.Errorf("there're labels not exist") + } + //check resource exist + m := make(map[string][]*mongodb.LabelBinding) + for _, binding := range cr.LabelBindings { + m[binding.Resource.Type] = append(m[binding.Resource.Type], binding) + } + for k, v := range m { + switch k { + case string(config.ResourceTypeWorkflow): + workflows := []commondb.Workflow{} + resourceSet := sets.NewString() + for _, vv := range v { + workflow := commondb.Workflow{ + Name: vv.Resource.Name, + ProjectName: vv.Resource.ProjectName, + } + workflows = append(workflows, workflow) + resourceSet.Insert(fmt.Sprintf("%s-%s", vv.Resource.Name, vv.Resource.ProjectName)) + } + wks, err := commondb.NewWorkflowColl().ListByWorkflows(commondb.ListWorkflowOpt{workflows}) + if err != nil { + logger.Errorf("can not find related resource err:%s", err) + return err + } + if len(wks) != resourceSet.Len() { + logger.Errorf("there're resources not exist") + return e.ErrForbidden.AddDesc("there're resources not exist") + } + case string(config.ResourceTypeEnvironment): + products := []commondb.Product{} + resourceSet := sets.NewString() + for _, vv := range v { + product := commondb.Product{ + Name: vv.Resource.Name, + ProjectName: vv.Resource.ProjectName, + } + products = append(products, product) + resourceSet.Insert(fmt.Sprintf("%s-%s", vv.Resource.Name, vv.Resource.ProjectName)) + } + pros, err := commondb.NewProductColl().ListByProducts(commondb.ListProductOpt{products}) + if err != nil { + logger.Errorf("can not find related resource err:%s", err) + return err + } + if len(pros) != resourceSet.Len() { + logger.Errorf("there're resources not exist") + return e.ErrForbidden.AddDesc("there're resources not exist") + } + } + } + + return mongodb.NewLabelBindingColl().CreateMany(cr.LabelBindings) +} + +type DeleteLabelBindingsArgs struct { + LabelBindings []*mongodb.LabelBinding `json:"label_bindings"` +} + +func DeleteLabelBindings(dr *DeleteLabelBindingsArgs, userName string, logger *zap.SugaredLogger) error { + logger.Infof("DeleteLabelBindings:%v userName:%s", dr, userName) + return mongodb.NewLabelBindingColl().BulkDelete(dr.LabelBindings) +} + +type DeleteLabelsBindingsByIdsArgs struct { + IDs []string +} + +func DeleteLabelsBindingsByIds(ids []string) error { + return mongodb.NewLabelBindingColl().BulkDeleteByIds(ids) +} diff --git a/pkg/microservice/aslan/core/log/handler/log.go b/pkg/microservice/aslan/core/log/handler/log.go index 6627420d3102d0e3780fe5c3ec8fb55aede90c12..e1cdda7a3dd504e1129fc775e7d2be18cec3bf40 100644 --- a/pkg/microservice/aslan/core/log/handler/log.go +++ b/pkg/microservice/aslan/core/log/handler/log.go @@ -18,6 +18,7 @@ package handler import ( "context" + "fmt" "strconv" "strings" @@ -37,7 +38,7 @@ func GetBuildJobContainerLogs(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid task id") return } - // job名称使用全小写，避免出现subdomain错误 + // Use all lowercase job names to avoid subdomain errors ctx.Resp, ctx.Err = logservice.GetBuildJobContainerLogs( c.Param("pipelineName"), c.Param("serviceName"), @@ -55,10 +56,23 @@ func GetWorkflowBuildJobContainerLogs(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid task id") return } - // job名称使用全小写，避免出现subdomain错误 + // Use all lowercase job names to avoid subdomain errors ctx.Resp, ctx.Err = logservice.GetWorkflowBuildJobContainerLogs(strings.ToLower(c.Param("pipelineName")), c.Param("serviceName"), c.Query("type"), taskID, ctx.Logger) } +func GetWorkflowV4JobContainerLogs(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + taskID, err := strconv.ParseInt(c.Param("taskID"), 10, 64) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid task id") + return + } + // Use all lowercase job names to avoid subdomain errors + ctx.Resp, ctx.Err = logservice.GetWorkflowV4JobContainerLogs(strings.ToLower(c.Param("workflowName")), c.Param("jobName"), taskID, ctx.Logger) +} + func GetTestJobContainerLogs(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() @@ -69,8 +83,8 @@ func GetTestJobContainerLogs(c *gin.Context) { return } - // job名称使用全小写，避免出现subdomain错误 - ctx.Resp, ctx.Err = logservice.GetTestJobContainerLogs(c.Param("pipelineName"), c.Param("testName"), taskID, ctx.Logger) + // Use all lowercase job names to avoid subdomain errors + ctx.Resp, ctx.Err = logservice.GetTestJobContainerLogs(strings.ToLower(c.Param("pipelineName")), c.Param("testName"), taskID, ctx.Logger) } func GetWorkflowTestJobContainerLogs(c *gin.Context) { @@ -83,8 +97,8 @@ func GetWorkflowTestJobContainerLogs(c *gin.Context) { return } - // job名称使用全小写，避免出现subdomain错误 - ctx.Resp, ctx.Err = logservice.GetWorkflowTestJobContainerLogs(c.Param("pipelineName"), c.Param("serviceName"), c.Query("workflowType"), taskID, ctx.Logger) + // Use all lowercase job names to avoid subdomain errors + ctx.Resp, ctx.Err = logservice.GetWorkflowTestJobContainerLogs(strings.ToLower(c.Param("pipelineName")), c.Param("serviceName"), c.Query("workflowType"), taskID, ctx.Logger) } func GetContainerLogs(c *gin.Context) { @@ -94,7 +108,7 @@ func GetContainerLogs(c *gin.Context) { podName := c.Param("name") containerName := c.Query("container") envName := c.Query("envName") - productName := c.Query("productName") + productName := c.Query("projectName") tailLines, err := strconv.ParseInt(c.Query("tailLines"), 10, 64) if err != nil { @@ -115,3 +129,41 @@ func GetContainerLogs(c *gin.Context) { logservice.ContainerLogStream(ctx1, streamChan, envName, productName, podName, containerName, follow, tailLines, ctx.Logger) }, ctx.Logger) } + +func GetWorkflowBuildV3JobContainerLogs(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + taskID, err := strconv.ParseInt(c.Param("taskId"), 10, 64) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid task id") + return + } + // Use all lowercase job names to avoid subdomain errors + ctx.Resp, ctx.Err = logservice.GetWorkflowBuildV3JobContainerLogs(strings.ToLower(c.Param("workflowName")), c.Query("type"), taskID, ctx.Logger) +} + +func GetScanningContainerLogs(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + id := c.Param("id") + if id == "" { + ctx.Err = fmt.Errorf("id must be provided") + return + } + + taskIDStr := c.Param("scan_id") + if taskIDStr == "" { + ctx.Err = fmt.Errorf("scan_id must be provided") + return + } + + taskID, err := strconv.ParseInt(taskIDStr, 10, 64) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid task id") + return + } + + ctx.Resp, ctx.Err = logservice.GetScanningContainerLogs(id, taskID, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/log/handler/router.go b/pkg/microservice/aslan/core/log/handler/router.go index 03bda972111b6c2fda3edd04901c7f3dc59340e2..aa762298810ac49d604c7a0ba516a348fd519891 100644 --- a/pkg/microservice/aslan/core/log/handler/router.go +++ b/pkg/microservice/aslan/core/log/handler/router.go @@ -18,15 +18,11 @@ package handler import ( "github.com/gin-gonic/gin" - - gin2 "github.com/koderover/zadig/pkg/middleware/gin" ) type Router struct{} func (*Router) Inject(router *gin.RouterGroup) { - router.Use(gin2.Auth()) - { router.GET("/pods/:name", GetContainerLogs) } @@ -37,6 +33,9 @@ func (*Router) Inject(router *gin.RouterGroup) { log.GET("/workflow/:pipelineName/tasks/:taskId/service/:serviceName", GetWorkflowBuildJobContainerLogs) log.GET("/pipelines/:pipelineName/tasks/:taskId/tests/:testName", GetTestJobContainerLogs) log.GET("/workflow/:pipelineName/tasks/:taskId/tests/:testName/service/:serviceName", GetWorkflowTestJobContainerLogs) + log.GET("/v3/workflow/:workflowName/tasks/:taskId", GetWorkflowBuildV3JobContainerLogs) + log.GET("/scanning/:id/task/:scan_id", GetScanningContainerLogs) + log.GET("/v4/workflow/:workflowName/tasks/:taskID/jobs/:jobName", GetWorkflowV4JobContainerLogs) } sse := router.Group("sse") @@ -47,5 +46,8 @@ func (*Router) Inject(router *gin.RouterGroup) { sse.GET("/test/:pipelineName/:taskId/:testName/:lines", GetTestJobContainerLogsSSE) sse.GET("/workflow/test/:pipelineName/:taskId/:testName/:lines/:serviceName", GetWorkflowTestJobContainerLogsSSE) sse.GET("/service/build/:serviceName/:envName/:productName", GetServiceJobContainerLogsSSE) + sse.GET("/v3/workflow/build/:workflowName/:taskId/:lines", GetWorkflowBuildV3JobContainerLogsSSE) + sse.GET("/scanning/:id/task/:scan_id", GetScanningContainerLogsSSE) + sse.GET("/v4/workflow/:workflowName/:taskID/:jobName/:lines", GetWorkflowJobContainerLogsSSE) } } diff --git a/pkg/microservice/aslan/core/log/handler/sse.go b/pkg/microservice/aslan/core/log/handler/sse.go index 7293d4360b64d8b3088212e5c908770e536f759c..4813fe971d96153dd457a7e72ed89d2fc5d98724 100644 --- a/pkg/microservice/aslan/core/log/handler/sse.go +++ b/pkg/microservice/aslan/core/log/handler/sse.go @@ -18,6 +18,7 @@ package handler import ( "context" + "fmt" "strconv" "github.com/gin-contrib/sse" @@ -25,6 +26,7 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" logservice "github.com/koderover/zadig/pkg/microservice/aslan/core/log/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/testing/service" internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/util/ginzap" @@ -39,7 +41,7 @@ func GetContainerLogsSSE(c *gin.Context) { } envName := c.Query("envName") - productName := c.Query("productName") + productName := c.Query("projectName") internalhandler.Stream(c, func(ctx context.Context, streamChan chan interface{}) { logservice.ContainerLogStream(ctx, streamChan, envName, productName, c.Param("podName"), c.Param("containerName"), true, tails, logger) @@ -77,6 +79,35 @@ func GetBuildJobContainerLogsSSE(c *gin.Context) { }, ctx.Logger) } +func GetWorkflowJobContainerLogsSSE(c *gin.Context) { + ctx := internalhandler.NewContext(c) + + taskID, err := strconv.ParseInt(c.Param("taskID"), 10, 64) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid task id") + internalhandler.JSONResponse(c, ctx) + return + } + + tails, err := strconv.ParseInt(c.Param("lines"), 10, 64) + if err != nil { + tails = int64(10) + } + + internalhandler.Stream(c, func(ctx1 context.Context, streamChan chan interface{}) { + logservice.WorkflowTaskV4ContainerLogStream( + ctx1, streamChan, + &logservice.GetContainerOptions{ + Namespace: config.Namespace(), + PipelineName: c.Param("workflowName"), + SubTask: c.Param("jobName"), + TaskID: taskID, + TailLines: tails, + }, + ctx.Logger) + }, ctx.Logger) +} + func GetWorkflowBuildJobContainerLogsSSE(c *gin.Context) { ctx := internalhandler.NewContext(c) @@ -94,15 +125,16 @@ func GetWorkflowBuildJobContainerLogsSSE(c *gin.Context) { subTask := c.Query("subTask") options := &logservice.GetContainerOptions{ - Namespace: config.Namespace(), - PipelineName: c.Param("pipelineName"), - SubTask: subTask, - TailLines: tails, - TaskID: taskID, - ServiceName: c.Param("serviceName"), - PipelineType: string(config.WorkflowType), - EnvName: c.Query("envName"), - ProductName: c.Query("productName"), + Namespace: config.Namespace(), + PipelineName: c.Param("pipelineName"), + SubTask: subTask, + TailLines: tails, + TaskID: taskID, + ServiceName: c.Param("serviceName"), + ServiceModule: c.Query("serviceModule"), + PipelineType: string(config.WorkflowType), + EnvName: c.Query("envName"), + ProductName: c.Query("projectName"), } internalhandler.Stream(c, func(ctx1 context.Context, streamChan chan interface{}) { @@ -215,3 +247,85 @@ func GetServiceJobContainerLogsSSE(c *gin.Context) { ctx.Logger) }, ctx.Logger) } + +func GetWorkflowBuildV3JobContainerLogsSSE(c *gin.Context) { + ctx := internalhandler.NewContext(c) + + taskID, err := strconv.ParseInt(c.Param("taskId"), 10, 64) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid task id") + internalhandler.JSONResponse(c, ctx) + return + } + + tails, err := strconv.ParseInt(c.Param("lines"), 10, 64) + if err != nil { + tails = int64(10) + } + + subTask := c.Query("subTask") + options := &logservice.GetContainerOptions{ + Namespace: config.Namespace(), + PipelineName: c.Param("workflowName"), + SubTask: subTask, + TailLines: tails, + TaskID: taskID, + PipelineType: string(config.WorkflowTypeV3), + EnvName: c.Query("envName"), + ProductName: c.Query("projectName"), + ServiceName: fmt.Sprintf("%s-job", c.Param("workflowName")), + } + + internalhandler.Stream(c, func(ctx1 context.Context, streamChan chan interface{}) { + logservice.TaskContainerLogStream( + ctx1, streamChan, + options, + ctx.Logger) + }, ctx.Logger) +} + +func GetScanningContainerLogsSSE(c *gin.Context) { + ctx := internalhandler.NewContext(c) + + id := c.Param("id") + if id == "" { + ctx.Err = fmt.Errorf("id must be provided") + return + } + + taskIDStr := c.Param("scan_id") + if taskIDStr == "" { + ctx.Err = fmt.Errorf("scan_id must be provided") + return + } + + taskID, err := strconv.ParseInt(taskIDStr, 10, 64) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid task id") + return + } + + tails, err := strconv.ParseInt(c.Query("lines"), 10, 64) + if err != nil { + tails = int64(10) + } + + resp, err := service.GetScanningModuleByID(id, ctx.Logger) + scanningName := fmt.Sprintf("%s-%s-%s", resp.Name, id, "scanning-job") + options := &logservice.GetContainerOptions{ + Namespace: config.Namespace(), + PipelineName: scanningName, + SubTask: string(config.TaskScanning), + TailLines: tails, + TaskID: taskID, + PipelineType: string(config.ScanningType), + ServiceName: resp.Name, + } + + internalhandler.Stream(c, func(ctx1 context.Context, streamChan chan interface{}) { + logservice.TaskContainerLogStream( + ctx1, streamChan, + options, + ctx.Logger) + }, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/log/service/log.go b/pkg/microservice/aslan/core/log/service/log.go index 7d8c5081ba0de99a4f74fa7290544d9332aa684c..e7f6f13854fef2743c018f1f7f478dee1aae6114 100644 --- a/pkg/microservice/aslan/core/log/service/log.go +++ b/pkg/microservice/aslan/core/log/service/log.go @@ -55,6 +55,15 @@ func GetWorkflowBuildJobContainerLogs(pipelineName, serviceName, buildType strin return buildLog, nil } +func GetWorkflowV4JobContainerLogs(workflowName, jobName string, taskID int64, log *zap.SugaredLogger) (string, error) { + buildJobNamePrefix := jobName + buildLog, err := getContainerLogFromS3(workflowName, buildJobNamePrefix, taskID, log) + if err != nil { + return "", err + } + return buildLog, nil +} + func GetTestJobContainerLogs(pipelineName, serviceName string, taskID int64, log *zap.SugaredLogger) (string, error) { taskName := fmt.Sprintf("%s-%s-%d-%s-%s", config.SingleType, pipelineName, taskID, config.TaskTestingV2, serviceName) return getContainerLogFromS3(pipelineName, taskName, taskID, log) @@ -72,6 +81,7 @@ func GetWorkflowTestJobContainerLogs(pipelineName, serviceName, pipelineType str func getContainerLogFromS3(pipelineName, filenamePrefix string, taskID int64, log *zap.SugaredLogger) (string, error) { fileName := strings.Replace(strings.ToLower(filenamePrefix), "_", "-", -1) + fileName += ".log" tempFile, _ := util.GenerateTmpFile() defer func() { _ = os.Remove(tempFile) @@ -97,17 +107,11 @@ func getContainerLogFromS3(pipelineName, filenamePrefix string, taskID int64, lo log.Errorf("Failed to create s3 client, the error is: %+v", err) return "", err } - objectPrefix := storage.GetObjectPath(fileName) - fileList, err := client.ListFiles(storage.Bucket, objectPrefix, false) - if err != nil { - log.Errorf("GetContainerLogFromS3 ListFiles err:%v", err) - return "", err - } - if len(fileList) == 0 { - return "", nil - } - objectKey := storage.GetObjectPath(fileList[0]) - err = client.Download(storage.Bucket, objectKey, tempFile) + fullPath := storage.GetObjectPath(fileName) + err = client.DownloadWithOption(storage.Bucket, fullPath, tempFile, &s3tool.DownloadOption{ + IgnoreNotExistError: true, + RetryNum: 3, + }) if err != nil { log.Errorf("GetContainerLogFromS3 Download err:%v", err) return "", err @@ -142,3 +146,29 @@ func GetCurrentContainerLogs(podName, containerName, envName, productName string return buf.String(), nil } + +func GetWorkflowBuildV3JobContainerLogs(workflowName, buildType string, taskID int64, log *zap.SugaredLogger) (string, error) { + buildJobNamePrefix := fmt.Sprintf("%s-%s-%d-%s-%s", config.WorkflowTypeV3, workflowName, taskID, buildType, fmt.Sprintf("%s-job", workflowName)) + buildLog, err := getContainerLogFromS3(workflowName, buildJobNamePrefix, taskID, log) + if err != nil { + return "", err + } + + return buildLog, nil +} + +func GetScanningContainerLogs(scanID string, taskID int64, log *zap.SugaredLogger) (string, error) { + scanning, err := commonrepo.NewScanningColl().GetByID(scanID) + if err != nil { + log.Errorf("failed to get scanning from db to get scanning detail, the error is: %s", err) + return "", err + } + scanningName := fmt.Sprintf("%s-%s-%s", scanning.Name, scanID, "scanning-job") + scanningLogFilePrefix := fmt.Sprintf("%s-%s-%d-%s", config.ScanningType, scanningName, taskID, config.ScanningType) + buildLog, err := getContainerLogFromS3(scanningName, scanningLogFilePrefix, taskID, log) + if err != nil { + return "", err + } + + return buildLog, nil +} diff --git a/pkg/microservice/aslan/core/log/service/sse.go b/pkg/microservice/aslan/core/log/service/sse.go index eb7872dd3f3c0d74a72efac60f4419204a8d69ea..e2f6fa53070f6653d001a31a823a383637ea36cd 100644 --- a/pkg/microservice/aslan/core/log/service/sse.go +++ b/pkg/microservice/aslan/core/log/service/sse.go @@ -32,10 +32,11 @@ import ( commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" - krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" "github.com/koderover/zadig/pkg/tool/kube/containerlog" "github.com/koderover/zadig/pkg/tool/kube/getter" "github.com/koderover/zadig/pkg/tool/kube/watcher" + "github.com/koderover/zadig/pkg/util" ) const ( @@ -43,16 +44,18 @@ const ( ) type GetContainerOptions struct { - Namespace string - PipelineName string - SubTask string - TailLines int64 - TaskID int64 - PipelineType string - ServiceName string - TestName string - EnvName string - ProductName string + Namespace string + PipelineName string + SubTask string + TailLines int64 + TaskID int64 + PipelineType string + ServiceName string + ServiceModule string + TestName string + EnvName string + ProductName string + ClusterID string } func ContainerLogStream(ctx context.Context, streamChan chan interface{}, envName, productName, podName, containerName string, follow bool, tailLines int64, log *zap.SugaredLogger) { @@ -114,17 +117,76 @@ func containerLogStream(ctx context.Context, streamChan chan interface{}, namesp } } +func parseServiceName(fullServiceName, serviceModule string) (string, string) { + // when service module is passed, use the passed value + // otherwise we fall back to the old logic + if len(serviceModule) > 0 { + return strings.TrimPrefix(fullServiceName, serviceModule+"_"), serviceModule + } + var serviceName string + serviceNames := strings.Split(fullServiceName, "_") + switch len(serviceNames) { + case 1: + serviceModule = serviceNames[0] + case 2: + // Note: Starting from V1.10.0, this field will be in the format of `ServiceModule_ServiceName`. + serviceModule = serviceNames[0] + serviceName = serviceNames[1] + } + return serviceName, serviceModule +} + func TaskContainerLogStream(ctx context.Context, streamChan chan interface{}, options *GetContainerOptions, log *zap.SugaredLogger) { if options == nil { return } log.Debugf("Start to get task container log.") - if options.EnvName != "" && options.ProductName != "" { - //修改pipelineName，判断pipelineName是否为空，为空代表是来自环境里面请求，不为空代表是来自工作流任务的请求 - if options.PipelineName == "" { - options.PipelineName = fmt.Sprintf("%s-%s-%s", options.ServiceName, options.EnvName, "job") - if taskObj, err := commonrepo.NewTaskColl().FindTask(options.PipelineName, config.ServiceType); err == nil { - options.TaskID = taskObj.TaskID + + serviceName, serviceModule := parseServiceName(options.ServiceName, options.ServiceModule) + + // Cloud host scenario reads real-time logs from the environment, so pipelineName is empty. + if options.EnvName != "" && options.ProductName != "" && options.PipelineName == "" { + // Modify pipelineName to check whether pipelineName is empty: + // - Empty pipelineName indicates requests from the environment + // - Non-empty pipelineName indicate requests from workflow tasks + options.PipelineName = fmt.Sprintf("%s-%s-%s", serviceName, options.EnvName, "job") + if taskObj, err := commonrepo.NewTaskColl().FindTask(options.PipelineName, config.ServiceType); err == nil { + options.TaskID = taskObj.TaskID + } + } else if options.ProductName != "" { + buildFindOptions := &commonrepo.BuildFindOption{ + ProductName: options.ProductName, + Targets: []string{serviceModule}, + } + if serviceName != "" { + buildFindOptions.ServiceName = serviceName + } + + build, err := commonrepo.NewBuildColl().Find(buildFindOptions) + if err != nil { + // Maybe this service is a shared service + buildFindOptions := &commonrepo.BuildFindOption{ + Targets: []string{serviceModule}, + } + if serviceName != "" { + buildFindOptions.ServiceName = serviceName + } + + build, err = commonrepo.NewBuildColl().Find(buildFindOptions) + if err != nil { + log.Errorf("Failed to query build for service %s: %s", serviceName, err) + return + } + } + // Compatible with the situation where the old data has not been modified + if build != nil && build.PreBuild != nil && build.PreBuild.ClusterID != "" { + options.ClusterID = build.PreBuild.ClusterID + + switch build.PreBuild.ClusterID { + case setting.LocalClusterID: + options.Namespace = config.Namespace() + default: + options.Namespace = setting.AttachedClusterNamespace } } } @@ -136,25 +198,65 @@ func TaskContainerLogStream(ctx context.Context, streamChan chan interface{}, op waitAndGetLog(ctx, streamChan, selector, options, log) } -func TestJobContainerLogStream(ctx context.Context, streamChan chan interface{}, options *GetContainerOptions, log *zap.SugaredLogger) { +func WorkflowTaskV4ContainerLogStream(ctx context.Context, streamChan chan interface{}, options *GetContainerOptions, log *zap.SugaredLogger) { + if options == nil { + return + } + log.Debugf("Start to get task container log.") + selector := getWorkflowSelector(options) + waitAndGetLog(ctx, streamChan, selector, options, log) +} + +func TestJobContainerLogStream(ctx context.Context, streamChan chan interface{}, options *GetContainerOptions, log *zap.SugaredLogger) { options.SubTask = string(config.TaskTestingV2) selector := getPipelineSelector(options) + // get cluster ID + testing, _ := commonrepo.NewTestingColl().Find(getTestName(options.ServiceName), "") + // Compatible with the situation where the old data has not been modified + if testing != nil && testing.PreTest != nil && testing.PreTest.ClusterID != "" { + options.ClusterID = testing.PreTest.ClusterID + + switch testing.PreTest.ClusterID { + case setting.LocalClusterID: + options.Namespace = config.Namespace() + default: + options.Namespace = setting.AttachedClusterNamespace + } + } waitAndGetLog(ctx, streamChan, selector, options, log) } +func getTestName(serviceName string) string { + testName := strings.TrimRight(serviceName, "-job") + return testName +} + func waitAndGetLog(ctx context.Context, streamChan chan interface{}, selector labels.Selector, options *GetContainerOptions, log *zap.SugaredLogger) { PodCtx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - log.Debugf("Waiting until pod is running before establishing the stream.") - err := watcher.WaitUntilPodRunning(PodCtx, options.Namespace, selector, krkubeclient.Clientset()) + log.Debugf("Waiting until pod is running before establishing the stream. labelSelector: %+v", selector) + clientSet, err := kubeclient.GetClientset(config.HubServerAddress(), options.ClusterID) + if err != nil { + log.Errorf("GetContainerLogs, get client set error: %s", err) + return + } + + err = watcher.WaitUntilPodRunning(PodCtx, options.Namespace, selector, clientSet) + if err != nil { + log.Errorf("GetContainerLogs, wait pod running error: %s", err) + return + } + + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), options.ClusterID) if err != nil { - log.Errorf("GetContainerLogs, wait pod running error: %+v", err) + log.Errorf("GetContainerLogs, get kube client error: %s", err) return } - pods, err := getter.ListPods(options.Namespace, selector, krkubeclient.Client()) + + pods, err := getter.ListPods(options.Namespace, selector, kubeClient) if err != nil { log.Errorf("GetContainerLogs, get pod error: %+v", err) return @@ -169,7 +271,7 @@ func waitAndGetLog(ctx context.Context, streamChan chan interface{}, selector la pods[0].Name, options.SubTask, true, options.TailLines, - krkubeclient.Clientset(), + clientSet, log, ) } @@ -186,7 +288,7 @@ func getPipelineSelector(options *GetContainerOptions) labels.Selector { ret[setting.TypeLabel] = options.SubTask if options.ServiceName != "" { - ret[setting.ServiceLabel] = strings.ToLower(options.ServiceName) + ret[setting.ServiceLabel] = strings.ToLower(util.ReturnValidLabelValue(options.ServiceName)) } if options.PipelineType != "" { @@ -195,3 +297,17 @@ func getPipelineSelector(options *GetContainerOptions) labels.Selector { return ret.AsSelector() } + +func getWorkflowSelector(options *GetContainerOptions) labels.Selector { + retMap := map[string]string{ + setting.JobLabelTaskKey: fmt.Sprintf("%s-%d", strings.ToLower(options.PipelineName), options.TaskID), + setting.JobLabelNameKey: strings.Replace(options.SubTask, "_", "-", -1), + } + // no need to add labels with empty value to a job + for k, v := range retMap { + if len(v) == 0 { + delete(retMap, k) + } + } + return labels.Set(retMap).AsSelector() +} diff --git a/pkg/microservice/aslan/core/multicluster/handler/bundle.go b/pkg/microservice/aslan/core/multicluster/handler/bundle.go new file mode 100644 index 0000000000000000000000000000000000000000..1c67c3ef5617eb53b349302be244af0a88213ff6 --- /dev/null +++ b/pkg/microservice/aslan/core/multicluster/handler/bundle.go @@ -0,0 +1,31 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/multicluster/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" +) + +func GetBundleResources(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.GetBundleResources() +} diff --git a/pkg/microservice/aslan/core/multicluster/handler/cache.go b/pkg/microservice/aslan/core/multicluster/handler/cache.go new file mode 100644 index 0000000000000000000000000000000000000000..3714033a0df5b9bcc2daea11b545eee77786a6ab --- /dev/null +++ b/pkg/microservice/aslan/core/multicluster/handler/cache.go @@ -0,0 +1,39 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/multicluster/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + "github.com/koderover/zadig/pkg/types" +) + +func ListStorageClasses(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.ListStorageClasses(c, c.Param("id"), types.StorageClassType(c.Query("type"))) +} + +func ListPVCs(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.ListPVCs(c, c.Param("id"), c.Param("namespace")) +} diff --git a/pkg/microservice/aslan/core/multicluster/handler/clusters.go b/pkg/microservice/aslan/core/multicluster/handler/clusters.go index c421e307e5722f9b917a8975e0259b8d9d94dccc..38ed4baad1db9bcfcf81801c68ef8e45414d40c7 100644 --- a/pkg/microservice/aslan/core/multicluster/handler/clusters.go +++ b/pkg/microservice/aslan/core/multicluster/handler/clusters.go @@ -22,19 +22,23 @@ import ( "github.com/gin-gonic/gin" - commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/multicluster/service" internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/log" ) func ListClusters(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = service.ListClusters(c.Query("clusterType"), - ctx.Logger, - ) + clusters, found := internalhandler.GetResourcesInHeader(c) + if found && len(clusters) == 0 { + ctx.Resp = []*service.K8SCluster{} + return + } + + ctx.Resp, ctx.Err = service.ListClusters(clusters, c.Query("projectName"), ctx.Logger) } func GetCluster(c *gin.Context) { @@ -48,7 +52,7 @@ func CreateCluster(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - args := new(commonmodels.K8SCluster) + args := new(service.K8SCluster) if err := c.BindJSON(args); err != nil { ctx.Err = e.ErrInvalidParam.AddErr(err) return @@ -60,7 +64,7 @@ func CreateCluster(c *gin.Context) { } args.CreatedAt = time.Now().Unix() - args.CreatedBy = ctx.Username + args.CreatedBy = ctx.UserName ctx.Resp, ctx.Err = service.CreateCluster(args, ctx.Logger) } @@ -69,14 +73,16 @@ func UpdateCluster(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - args := new(commonmodels.K8SCluster) + args := new(service.K8SCluster) if err := c.BindJSON(args); err != nil { ctx.Err = e.ErrInvalidParam.AddErr(err) + log.Errorf("Failed to bind data: %s", err) return } if err := args.Clean(); err != nil { ctx.Err = e.ErrInvalidParam.AddErr(err) + log.Errorf("Failed to clean args: %s", err) return } @@ -87,21 +93,21 @@ func DeleteCluster(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Err = service.DeleteCluster(ctx.Username, c.Param("id"), ctx.Logger) + ctx.Err = service.DeleteCluster(ctx.UserName, c.Param("id"), ctx.Logger) } func DisconnectCluster(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Err = service.DisconnectCluster(ctx.Username, c.Param("id"), ctx.Logger) + ctx.Err = service.DisconnectCluster(ctx.UserName, c.Param("id"), ctx.Logger) } func ReconnectCluster(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Err = service.ReconnectCluster(ctx.Username, c.Param("id"), ctx.Logger) + ctx.Err = service.ReconnectCluster(ctx.UserName, c.Param("id"), ctx.Logger) } func ClusterConnectFromAgent(c *gin.Context) { @@ -137,3 +143,17 @@ func GetClusterYaml(hubURI string) func(*gin.Context) { c.Abort() } } + +func UpgradeAgent(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Err = service.UpgradeAgent(c.Param("id"), ctx.Logger) +} + +func CheckEphemeralContainers(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.CheckEphemeralContainers(c, c.Param("id")) +} diff --git a/pkg/microservice/aslan/core/multicluster/handler/router.go b/pkg/microservice/aslan/core/multicluster/handler/router.go index fc4ba73703931f12b785a57fa759de0cad7601e1..cf559894b8afacbc677f26477ca84c8370fc82ae 100644 --- a/pkg/microservice/aslan/core/multicluster/handler/router.go +++ b/pkg/microservice/aslan/core/multicluster/handler/router.go @@ -18,8 +18,6 @@ package handler import ( "github.com/gin-gonic/gin" - - gin2 "github.com/koderover/zadig/pkg/middleware/gin" ) type Router struct{} @@ -29,19 +27,33 @@ func (*Router) Inject(router *gin.RouterGroup) { Agent := router.Group("agent") { Agent.GET("/:id/agent.yaml", GetClusterYaml("/api/hub")) + Agent.GET("/:id/upgrade", UpgradeAgent) } - router.Use(gin2.Auth()) - Cluster := router.Group("clusters") { Cluster.GET("", ListClusters) Cluster.GET("/:id", GetCluster) - Cluster.POST("", gin2.RequireSuperAdminAuth, CreateCluster) - Cluster.PUT("/:id", gin2.RequireSuperAdminAuth, UpdateCluster) - Cluster.DELETE("/:id", gin2.RequireSuperAdminAuth, DeleteCluster) - Cluster.PUT("/:id/disconnect", gin2.RequireSuperAdminAuth, DisconnectCluster) - Cluster.PUT("/:id/reconnect", gin2.RequireSuperAdminAuth, ReconnectCluster) + Cluster.POST("", CreateCluster) + Cluster.PUT("/:id", UpdateCluster) + Cluster.DELETE("/:id", DeleteCluster) + Cluster.PUT("/:id/disconnect", DisconnectCluster) + Cluster.PUT("/:id/reconnect", ReconnectCluster) + } + + bundles := router.Group("bundle-resources") + { + bundles.GET("", GetBundleResources) + } + + istio := router.Group("istio") + { + istio.GET("/check/:id", CheckIstiod) } + + router.GET("/:id/storageclasses", ListStorageClasses) + router.GET("/:id/:namespace/pvcs", ListPVCs) + + router.GET("/:id/check/ephemeralcontainers", CheckEphemeralContainers) } diff --git a/pkg/microservice/aslan/core/multicluster/handler/share_env.go b/pkg/microservice/aslan/core/multicluster/handler/share_env.go new file mode 100644 index 0000000000000000000000000000000000000000..8a187babc3731e6540a4ae60c6650705d42e420a --- /dev/null +++ b/pkg/microservice/aslan/core/multicluster/handler/share_env.go @@ -0,0 +1,30 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + "github.com/koderover/zadig/pkg/microservice/aslan/core/multicluster/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" +) + +func CheckIstiod(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.CheckIstiod(c, c.Param("id")) +} diff --git a/pkg/microservice/aslan/core/multicluster/service/bundle.go b/pkg/microservice/aslan/core/multicluster/service/bundle.go new file mode 100644 index 0000000000000000000000000000000000000000..dea11a3a3c361e272ec27335d2c4bf9a808a0183 --- /dev/null +++ b/pkg/microservice/aslan/core/multicluster/service/bundle.go @@ -0,0 +1,50 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "strconv" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/tool/log" +) + +type resourceSpec struct { + ResourceID string `json:"resourceID"` + ProjectName string `json:"projectName"` + Spec []string `json:"spec"` +} + +func GetBundleResources() ([]*resourceSpec, error) { + clusters, err := mongodb.NewK8SClusterColl().List(nil) + if err != nil { + log.Errorf("Failed to list clusters in db, err: %s", err) + return nil, err + } + + var res []*resourceSpec + for _, cluster := range clusters { + res = append(res, &resourceSpec{ + ResourceID: cluster.ID.Hex(), + Spec: []string{ + "production:" + strconv.FormatBool(cluster.Production), + }, + }) + } + + return res, nil +} diff --git a/pkg/microservice/aslan/core/multicluster/service/cache.go b/pkg/microservice/aslan/core/multicluster/service/cache.go new file mode 100644 index 0000000000000000000000000000000000000000..7c8466409ba51e843e8c4ee7c848e2fe120d3bd0 --- /dev/null +++ b/pkg/microservice/aslan/core/multicluster/service/cache.go @@ -0,0 +1,107 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" + "github.com/koderover/zadig/pkg/types" +) + +type PVC struct { + Name string `json:"name"` + StorageSizeInBytes int64 `json:"storage_size_in_bytes"` +} + +func ListStorageClasses(ctx context.Context, clusterID string, scType types.StorageClassType) ([]string, error) { + kclient, err := kubeclient.GetKubeClient(config.HubServerAddress(), clusterID) + if err != nil { + return nil, fmt.Errorf("failed to get kube client: %s", err) + } + + var namespace string + switch clusterID { + case setting.LocalClusterID: + // For cluster-level resources, we need to explicitly configure the namespace to be empty. + namespace = "" + default: + namespace = setting.AttachedClusterNamespace + } + + scList := &storagev1.StorageClassList{} + err = kclient.List(ctx, scList, client.InNamespace(namespace)) + if err != nil { + return nil, fmt.Errorf("failed to list storageclasses: %s", err) + } + + storageclasses := make([]string, 0, len(scList.Items)) + for _, item := range scList.Items { + if scType != types.StorageClassAll && !StorageProvisioner(item.Provisioner).IsNFS() { + continue + } + + storageclasses = append(storageclasses, item.Name) + } + + return storageclasses, nil +} + +func ListPVCs(ctx context.Context, clusterID, namespace string) ([]PVC, error) { + kclient, err := kubeclient.GetKubeClient(config.HubServerAddress(), clusterID) + if err != nil { + return nil, fmt.Errorf("failed to get kube client: %s", err) + } + + switch clusterID { + case setting.LocalClusterID: + // For now caller may not know exactly which namespace Zadig is deployed to, so a correction is made here. + namespace = config.Namespace() + default: + if namespace != setting.AttachedClusterNamespace { + return nil, fmt.Errorf("invalid namespace in attached cluster: %s. Valid: %s", namespace, setting.AttachedClusterNamespace) + } + } + + pvcList := &corev1.PersistentVolumeClaimList{} + err = kclient.List(ctx, pvcList, client.InNamespace(namespace)) + if err != nil { + return nil, err + } + + pvcs := make([]PVC, 0, len(pvcList.Items)) + for _, item := range pvcList.Items { + // Don't support `corev1.PersistentVolumeBlock` because the PVC would be mounted on multiple nodes. + if item.Spec.VolumeMode != nil && *item.Spec.VolumeMode == corev1.PersistentVolumeBlock { + continue + } + + pvcs = append(pvcs, PVC{ + Name: item.Name, + StorageSizeInBytes: item.Spec.Resources.Requests.Storage().Value(), + }) + } + + return pvcs, nil +} diff --git a/pkg/microservice/aslan/core/multicluster/service/clusters.go b/pkg/microservice/aslan/core/multicluster/service/clusters.go index f5eb7b7cc3ebd7d6b04926eeeeec264891b0ddd0..22efd94e0a9cdc62bd9f04a0b2719ef3f1bcdc29 100644 --- a/pkg/microservice/aslan/core/multicluster/service/clusters.go +++ b/pkg/microservice/aslan/core/multicluster/service/clusters.go @@ -17,23 +17,153 @@ limitations under the License. package service import ( + "context" + "fmt" "net/http" + "regexp" + "strings" + "time" "github.com/gin-gonic/gin" + "github.com/hashicorp/go-multierror" "go.uber.org/zap" + "helm.sh/helm/v3/pkg/releaseutil" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/discovery" + "sigs.k8s.io/controller-runtime/pkg/client" configbase "github.com/koderover/zadig/pkg/config" "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" + "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/kube/serializer" + "github.com/koderover/zadig/pkg/tool/kube/updater" + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types" + "github.com/koderover/zadig/pkg/util" + "github.com/koderover/zadig/pkg/util/ginzap" ) -func ListClusters(clusterType string, logger *zap.SugaredLogger) ([]*commonmodels.K8SCluster, error) { - s, _ := kube.NewService("") +var namePattern = regexp.MustCompile(`^[0-9a-zA-Z_.-]{1,32}$`) + +type K8SCluster struct { + ID string `json:"id,omitempty"` + Name string `json:"name"` + Description string `json:"description"` + AdvancedConfig *AdvancedConfig `json:"advanced_config,omitempty"` + Status setting.K8SClusterStatus `json:"status"` + Production bool `json:"production"` + CreatedAt int64 `json:"createdAt"` + CreatedBy string `json:"createdBy"` + Provider int8 `json:"provider"` + Local bool `json:"local"` + Cache types.Cache `json:"cache"` + LastConnectionTime int64 `json:"last_connection_time"` + UpdateHubagentErrorMsg string `json:"update_hubagent_error_msg"` + DindCfg *commonmodels.DindCfg `json:"dind_cfg"` +} + +type AdvancedConfig struct { + Strategy string `json:"strategy,omitempty" bson:"strategy,omitempty"` + NodeLabels []string `json:"node_labels,omitempty" bson:"node_labels,omitempty"` + ProjectNames []string `json:"project_names" bson:"project_names"` +} + +func (k *K8SCluster) Clean() error { + k.Name = strings.TrimSpace(k.Name) + k.Description = strings.TrimSpace(k.Description) + + if !namePattern.MatchString(k.Name) { + return fmt.Errorf("The cluster name does not meet the rules") + } + + return nil +} + +func ListClusters(ids []string, projectName string, logger *zap.SugaredLogger) ([]*K8SCluster, error) { + idSet := sets.NewString(ids...) + cs, err := commonrepo.NewK8SClusterColl().List(&commonrepo.ClusterListOpts{IDs: idSet.UnsortedList()}) + if err != nil { + logger.Errorf("Failed to list clusters, err: %s", err) + return nil, err + } + + existClusterID := sets.NewString() + if projectName != "" { + projectClusterRelations, _ := commonrepo.NewProjectClusterRelationColl().List(&commonrepo.ProjectClusterRelationOption{ + ProjectName: projectName, + }) + for _, projectClusterRelation := range projectClusterRelations { + existClusterID.Insert(projectClusterRelation.ClusterID) + } + } + + var res []*K8SCluster + for _, c := range cs { + if projectName != "" && !existClusterID.Has(c.ID.Hex()) { + continue + } + + // If the advanced configuration is empty, the project information needs to be returned, + // otherwise the front end will report an error + if c.AdvancedConfig == nil { + c.AdvancedConfig = &commonmodels.AdvancedConfig{} + } + + var advancedConfig *AdvancedConfig + if c.AdvancedConfig != nil { + advancedConfig = &AdvancedConfig{ + Strategy: c.AdvancedConfig.Strategy, + NodeLabels: convertToNodeLabels(c.AdvancedConfig.NodeLabels), + ProjectNames: getProjectNames(c.ID.Hex(), logger), + } + } + + if c.DindCfg == nil { + c.DindCfg = &commonmodels.DindCfg{ + Replicas: kube.DefaultDindReplicas, + Resources: &commonmodels.Resources{ + Limits: &commonmodels.Limits{ + CPU: kube.DefaultDindLimitsCPU, + Memory: kube.DefaultDindLimitsMemory, + }, + }, + Storage: &commonmodels.DindStorage{ + Type: kube.DefaultDindStorageType, + }, + } + } + + res = append(res, &K8SCluster{ + ID: c.ID.Hex(), + Name: c.Name, + Description: c.Description, + Status: c.Status, + Production: c.Production, + CreatedBy: c.CreatedBy, + CreatedAt: c.CreatedAt, + Provider: c.Provider, + Local: c.Local, + AdvancedConfig: advancedConfig, + Cache: c.Cache, + LastConnectionTime: c.LastConnectionTime, + UpdateHubagentErrorMsg: c.UpdateHubagentErrorMsg, + DindCfg: c.DindCfg, + }) + } - return s.ListClusters(clusterType, logger) + return res, nil } func GetCluster(id string, logger *zap.SugaredLogger) (*commonmodels.K8SCluster, error) { @@ -42,16 +172,212 @@ func GetCluster(id string, logger *zap.SugaredLogger) (*commonmodels.K8SCluster, return s.GetCluster(id, logger) } -func CreateCluster(cluster *commonmodels.K8SCluster, logger *zap.SugaredLogger) (*commonmodels.K8SCluster, error) { - s, _ := kube.NewService("") +func getProjectNames(clusterID string, logger *zap.SugaredLogger) (projectNames []string) { + projectClusterRelations, err := commonrepo.NewProjectClusterRelationColl().List(&commonrepo.ProjectClusterRelationOption{ClusterID: clusterID}) + if err != nil { + logger.Errorf("Failed to list projectClusterRelation, err:%s", err) + return []string{} + } + for _, projectClusterRelation := range projectClusterRelations { + projectNames = append(projectNames, projectClusterRelation.ProjectName) + } + return projectNames +} - return s.CreateCluster(cluster, logger) +func convertToNodeLabels(nodeSelectorRequirements []*commonmodels.NodeSelectorRequirement) []string { + var nodeLabels []string + for _, nodeSelectorRequirement := range nodeSelectorRequirements { + for _, labelValue := range nodeSelectorRequirement.Value { + nodeLabels = append(nodeLabels, fmt.Sprintf("%s:%s", nodeSelectorRequirement.Key, labelValue)) + } + } + + return nodeLabels } -func UpdateCluster(id string, cluster *commonmodels.K8SCluster, logger *zap.SugaredLogger) (*commonmodels.K8SCluster, error) { - s, _ := kube.NewService("") +func convertToNodeSelectorRequirements(nodeLabels []string) []*commonmodels.NodeSelectorRequirement { + var nodeSelectorRequirements []*commonmodels.NodeSelectorRequirement + nodeLabelM := make(map[string][]string) + for _, nodeLabel := range nodeLabels { + if !strings.Contains(nodeLabel, ":") || len(strings.Split(nodeLabel, ":")) != 2 { + continue + } + key := strings.Split(nodeLabel, ":")[0] + value := strings.Split(nodeLabel, ":")[1] + nodeLabelM[key] = append(nodeLabelM[key], value) + } + for key, value := range nodeLabelM { + nodeSelectorRequirements = append(nodeSelectorRequirements, &commonmodels.NodeSelectorRequirement{ + Key: key, + Value: value, + Operator: corev1.NodeSelectorOpIn, + }) + } + return nodeSelectorRequirements +} + +func CreateCluster(args *K8SCluster, logger *zap.SugaredLogger) (*commonmodels.K8SCluster, error) { + s, err := kube.NewService("") + if err != nil { + return nil, fmt.Errorf("failed to new kube service: %s", err) + } + + var advancedConfig *commonmodels.AdvancedConfig + if args.AdvancedConfig != nil { + advancedConfig = &commonmodels.AdvancedConfig{ + Strategy: args.AdvancedConfig.Strategy, + NodeLabels: convertToNodeSelectorRequirements(args.AdvancedConfig.NodeLabels), + ProjectNames: args.AdvancedConfig.ProjectNames, + } + } + + // If user does not configure a cache for the cluster, object storage is used by default. + err = setClusterCache(args) + if err != nil { + return nil, fmt.Errorf("failed to set cache for cluster %s: %s", args.ID, err) + } + + // If user does not set a dind config for the cluster, set the default values. + err = setClusterDind(args) + if err != nil { + return nil, fmt.Errorf("failed to set dind args for cluster %s: %s", args.ID, err) + } + + cluster := &commonmodels.K8SCluster{ + Name: args.Name, + Description: args.Description, + AdvancedConfig: advancedConfig, + Status: args.Status, + Production: args.Production, + Provider: args.Provider, + CreatedAt: args.CreatedAt, + CreatedBy: args.CreatedBy, + Cache: args.Cache, + DindCfg: args.DindCfg, + } - return s.UpdateCluster(id, cluster, logger) + return s.CreateCluster(cluster, args.ID, logger) +} + +func UpdateCluster(id string, args *K8SCluster, logger *zap.SugaredLogger) (*commonmodels.K8SCluster, error) { + s, err := kube.NewService("") + if err != nil { + return nil, fmt.Errorf("failed to new kube service: %s", err) + } + + advancedConfig := new(commonmodels.AdvancedConfig) + if args.AdvancedConfig != nil { + advancedConfig.Strategy = args.AdvancedConfig.Strategy + advancedConfig.NodeLabels = convertToNodeSelectorRequirements(args.AdvancedConfig.NodeLabels) + // Delete all projects associated with clusterID + err := commonrepo.NewProjectClusterRelationColl().Delete(&commonrepo.ProjectClusterRelationOption{ClusterID: id}) + if err != nil { + logger.Errorf("Failed to delete projectClusterRelation err:%s", err) + } + for _, projectName := range args.AdvancedConfig.ProjectNames { + err = commonrepo.NewProjectClusterRelationColl().Create(&commonmodels.ProjectClusterRelation{ + ProjectName: projectName, + ClusterID: id, + CreatedBy: args.CreatedBy, + }) + if err != nil { + logger.Errorf("Failed to create projectClusterRelation err:%s", err) + } + } + } + + // If user does not configure a cache for the cluster, object storage is used by default. + err = setClusterCache(args) + if err != nil { + return nil, fmt.Errorf("failed to set cache for cluster %s: %s", id, err) + } + + // If user does not set a dind config for the cluster, set the default values. + err = setClusterDind(args) + if err != nil { + return nil, fmt.Errorf("failed to set dind args for cluster %s: %s", args.ID, err) + } + + // If the user chooses to use dynamically generated storage resources, the system automatically creates the PVC. + // TODO: If the PVC is not successfully bound to the PV, it is necessary to consider how to expose this abnormal information. + // Depends on product design. + if args.Cache.MediumType == types.NFSMedium && args.Cache.NFSProperties.ProvisionType == types.DynamicProvision { + kclient, err := kubeclient.GetKubeClient(config.HubServerAddress(), id) + if err != nil { + return nil, fmt.Errorf("failed to get kube client: %s", err) + } + + var namespace string + switch id { + case setting.LocalClusterID: + namespace = config.Namespace() + args.DindCfg = nil + default: + namespace = setting.AttachedClusterNamespace + } + + pvcName := fmt.Sprintf("cache-%s-%d", args.Cache.NFSProperties.StorageClass, args.Cache.NFSProperties.StorageSizeInGiB) + pvc := &corev1.PersistentVolumeClaim{} + err = kclient.Get(context.TODO(), client.ObjectKey{ + Name: pvcName, + Namespace: namespace, + }, pvc) + if err == nil { + logger.Infof("PVC %s eixsts in %s", pvcName, namespace) + args.Cache.NFSProperties.PVC = pvcName + } else if !apierrors.IsNotFound(err) { + return nil, fmt.Errorf("failed to find PVC %s in %s: %s", pvcName, namespace, err) + } else { + filesystemVolume := corev1.PersistentVolumeFilesystem + storageQuantity, err := resource.ParseQuantity(fmt.Sprintf("%dGi", args.Cache.NFSProperties.StorageSizeInGiB)) + if err != nil { + return nil, fmt.Errorf("failed to parse storage size: %d. err: %s", args.Cache.NFSProperties.StorageSizeInGiB, err) + } + + pvc = &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvcName, + Namespace: namespace, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + StorageClassName: &args.Cache.NFSProperties.StorageClass, + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteMany, + }, + VolumeMode: &filesystemVolume, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: storageQuantity, + }, + }, + }, + } + + err = kclient.Create(context.TODO(), pvc) + if err != nil { + return nil, fmt.Errorf("failed to create PVC %s in %s: %s", pvcName, namespace, err) + } + + logger.Infof("Successfully create PVC %s in %s", pvcName, namespace) + args.Cache.NFSProperties.PVC = pvcName + } + } + + cluster := &commonmodels.K8SCluster{ + Name: args.Name, + Description: args.Description, + AdvancedConfig: advancedConfig, + Production: args.Production, + Cache: args.Cache, + DindCfg: args.DindCfg, + } + + cluster, err = s.UpdateCluster(id, cluster, logger) + if err != nil { + return nil, fmt.Errorf("failed to update cluster %q: %s", id, err) + } + + return cluster, UpgradeAgent(id, logger) } func DeleteCluster(username, clusterID string, logger *zap.SugaredLogger) error { @@ -69,6 +395,10 @@ func DeleteCluster(username, clusterID string, logger *zap.SugaredLogger) error s, _ := kube.NewService("") + if err = commonrepo.NewProjectClusterRelationColl().Delete(&commonrepo.ProjectClusterRelationOption{ClusterID: clusterID}); err != nil { + logger.Errorf("Failed to delete projectClusterRelation err:%s", err) + } + return s.DeleteCluster(username, clusterID, logger) } @@ -93,5 +423,350 @@ func ProxyAgent(writer gin.ResponseWriter, request *http.Request) { func GetYaml(id, hubURI string, useDeployment bool, logger *zap.SugaredLogger) ([]byte, error) { s, _ := kube.NewService("") - return s.GetYaml(id, config.HubAgentImage(), configbase.SystemAddress(), hubURI, useDeployment, logger) + return s.GetYaml(id, config.HubAgentImage(), config.ResourceServerImage(), configbase.SystemAddress(), hubURI, useDeployment, logger) +} + +func UpgradeAgent(id string, logger *zap.SugaredLogger) error { + s, err := kube.NewService("") + if err != nil { + return fmt.Errorf("failed to new kube service: %s", err) + } + + clusterInfo, err := s.GetCluster(id, logger) + if err != nil { + return err + } + if clusterInfo.Status != setting.Normal { + return fmt.Errorf("cluster %s status %s not support Upgrade Agent", clusterInfo.Name, clusterInfo.Status) + } + + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), id) + if err != nil { + return fmt.Errorf("failed to get kube client: %s cluster: %s", err, clusterInfo.Name) + } + + // Upgrade local cluster. + if id == setting.LocalClusterID { + return UpgradeDind(kubeClient, clusterInfo, config.Namespace()) + } + + // Upgrade attached cluster. + yamls, err := s.GetYaml(id, config.HubAgentImage(), config.ResourceServerImage(), configbase.SystemAddress(), "/api/hub", true, logger) + if err != nil { + return err + } + + errList := new(multierror.Error) + manifests := releaseutil.SplitManifests(string(yamls)) + resources := make([]*unstructured.Unstructured, 0, len(manifests)) + for _, item := range manifests { + u, err := serializer.NewDecoder().YamlToUnstructured([]byte(item)) + if err != nil { + log.Errorf("[UpgradeAgent] Failed to convert yaml to Unstructured, manifest is\n%s\n, error: %v", item, err) + errList = multierror.Append(errList, err) + continue + } + resources = append(resources, u) + } + + for _, u := range resources { + if u.GetKind() == "StatefulSet" && u.GetName() == types.DindStatefulSetName { + err = UpgradeDind(kubeClient, clusterInfo, setting.AttachedClusterNamespace) + } else { + err = updater.CreateOrPatchUnstructured(u, kubeClient) + } + + if err != nil { + log.Errorf("[UpgradeAgent] Failed to create or update %s, manifest is\n%v\n, error: %v", u.GetKind(), u, err) + errList = multierror.Append(errList, err) + continue + } + } + updateHubagentErrorMsg := "" + if len(errList.Errors) > 0 { + updateHubagentErrorMsg = errList.Error() + } + + return s.UpdateUpgradeAgentInfo(id, updateHubagentErrorMsg) +} + +func setClusterCache(args *K8SCluster) error { + if args.Cache.MediumType != "" { + return nil + } + + defaultStorage, err := commonrepo.NewS3StorageColl().FindDefault() + if err != nil { + return fmt.Errorf("failed to find default object storage for cluster %s: %s", args.ID, err) + } + + // Since the attached cluster cannot access the minio in the local cluster, if the default object storage + // is minio, the attached cluster cannot be configured to use minio as a cache. + // + // Note: + // - Since the local cluster will be written to the database when Zadig is created, empty ID indicates + // that the cluster is attached. + // - Currently, we do not support attaching the local cluster again. + if (args.ID == "" || args.ID != setting.LocalClusterID) && strings.Contains(defaultStorage.Endpoint, ZadigMinioSVC) { + return nil + } + + args.Cache.MediumType = types.ObjectMedium + args.Cache.ObjectProperties = types.ObjectProperties{ + ID: defaultStorage.ID.Hex(), + } + + return nil +} + +func setClusterDind(cluster *K8SCluster) error { + if cluster.DindCfg == nil { + cluster.DindCfg = &commonmodels.DindCfg{ + Replicas: kube.DefaultDindReplicas, + Resources: &commonmodels.Resources{ + Limits: &commonmodels.Limits{ + CPU: kube.DefaultDindLimitsCPU, + Memory: kube.DefaultDindLimitsMemory, + }, + }, + Storage: &commonmodels.DindStorage{ + Type: kube.DefaultDindStorageType, + }, + } + + return nil + } + + if cluster.DindCfg.Replicas <= 0 { + cluster.DindCfg.Replicas = kube.DefaultDindReplicas + } + if cluster.DindCfg.Resources == nil { + cluster.DindCfg.Resources = &commonmodels.Resources{ + Limits: &commonmodels.Limits{ + CPU: kube.DefaultDindLimitsCPU, + Memory: kube.DefaultDindLimitsMemory, + }, + } + } + if cluster.DindCfg.Storage == nil { + cluster.DindCfg.Storage = &commonmodels.DindStorage{ + Type: kube.DefaultDindStorageType, + } + } + if cluster.DindCfg.Storage.Type == commonmodels.DindStorageRootfs { + cluster.DindCfg.Storage.StorageClass = "" + cluster.DindCfg.Storage.StorageSizeInGiB = 0 + } + + return nil +} + +func ClusterApplyUpgradeAgent() { + for i := 0; i < 3; i++ { + time.Sleep(10 * time.Second) + k8sClusters, err := commonrepo.NewK8SClusterColl().List(nil) + if err != nil && !commonrepo.IsErrNoDocuments(err) { + log.Errorf("[ClusterApplyUpgradeAgent] list cluster error:%s", err) + return + } + + for _, cluster := range k8sClusters { + if cluster.Status != setting.Normal { + log.Warnf("[ClusterApplyUpgradeAgent] not support apply. cluster %s status %s ", cluster.Name, cluster.Status) + continue + } + if err := UpgradeAgent(cluster.ID.Hex(), ginzap.WithContext(&gin.Context{}).Sugar()); err != nil { + log.Errorf("[ClusterApplyUpgradeAgent] UpgradeAgent cluster %s error:%s", cluster.Name, err) + } else { + log.Infof("[ClusterApplyUpgradeAgent] UpgradeAgent cluster %s successed", cluster.Name) + } + } + time.Sleep(20 * time.Second) + } +} + +func CheckEphemeralContainers(ctx context.Context, clusterID string) (bool, error) { + restConfig, err := kubeclient.GetRESTConfig(config.HubServerAddress(), clusterID) + if err != nil { + return false, fmt.Errorf("failed to get rest config: %s", err) + } + + discoveryClient, err := discovery.NewDiscoveryClientForConfig(restConfig) + if err != nil { + return false, fmt.Errorf("failed to new discovery client: %s", err) + } + + resource, err := discoveryClient.ServerResourcesForGroupVersion("v1") + if err != nil { + return false, fmt.Errorf("failed to get server resource: %s", err) + } + + for _, apiResource := range resource.APIResources { + if apiResource.Name == "pods/ephemeralcontainers" { + return true, nil + } + } + + return false, nil +} + +func UpgradeDind(kclient client.Client, cluster *commonmodels.K8SCluster, ns string) error { + if cluster.DindCfg == nil { + return nil + } + + ctx := context.TODO() + dindSts := &appsv1.StatefulSet{} + err := kclient.Get(ctx, client.ObjectKey{ + Name: types.DindStatefulSetName, + Namespace: ns, + }, dindSts) + if err != nil { + return fmt.Errorf("failed to get dind StatefulSet in local cluster: %s", err) + } + + dindSts.Spec.Replicas = util.GetInt32Pointer(int32(cluster.DindCfg.Replicas)) + + if cluster.DindCfg.Resources != nil && cluster.DindCfg.Resources.Limits != nil { + cpuSize := fmt.Sprintf("%dm", cluster.DindCfg.Resources.Limits.CPU) + cpuQuantity, err := resource.ParseQuantity(cpuSize) + if err != nil { + return fmt.Errorf("failed to parse CPU resource %q: %s", cpuSize, err) + } + + memSize := fmt.Sprintf("%dMi", cluster.DindCfg.Resources.Limits.Memory) + memQuantity, err := resource.ParseQuantity(memSize) + if err != nil { + return fmt.Errorf("failed to parse Memory resource %q: %s", memSize, err) + } + + for i, container := range dindSts.Spec.Template.Spec.Containers { + if container.Name != types.DindContainerName { + continue + } + + container.Resources.Limits = corev1.ResourceList{ + corev1.ResourceCPU: cpuQuantity, + corev1.ResourceMemory: memQuantity, + } + dindSts.Spec.Template.Spec.Containers[i] = container + } + } + + if cluster.DindCfg.Storage != nil { + switch cluster.DindCfg.Storage.Type { + case commonmodels.DindStorageRootfs: + for i, container := range dindSts.Spec.Template.Spec.Containers { + if container.Name != types.DindContainerName { + continue + } + + volumeMounts := []corev1.VolumeMount{} + for _, volumeMount := range container.VolumeMounts { + if volumeMount.MountPath == types.DindMountPath { + continue + } + + volumeMounts = append(volumeMounts, volumeMount) + } + container.VolumeMounts = volumeMounts + dindSts.Spec.Template.Spec.Containers[i] = container + } + + volumeClaimTemplates := []corev1.PersistentVolumeClaim{} + for _, volumeClaimTemplate := range dindSts.Spec.VolumeClaimTemplates { + if volumeClaimTemplate.Name == types.DindMountName { + continue + } + + volumeClaimTemplates = append(volumeClaimTemplates, volumeClaimTemplate) + } + dindSts.Spec.VolumeClaimTemplates = volumeClaimTemplates + default: + for i, container := range dindSts.Spec.Template.Spec.Containers { + if container.Name != types.DindContainerName { + continue + } + + var exists bool + for _, volumeMount := range container.VolumeMounts { + if volumeMount.MountPath == types.DindMountPath { + exists = true + break + } + } + + if exists { + break + } + + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ + Name: types.DindMountName, + MountPath: types.DindMountPath, + }) + dindSts.Spec.Template.Spec.Containers[i] = container + } + + var dockerStorageExists bool + for _, volumeClaimTemplate := range dindSts.Spec.VolumeClaimTemplates { + if volumeClaimTemplate.Name == types.DindMountName { + dockerStorageExists = true + break + } + } + + if !dockerStorageExists { + storageSize := fmt.Sprintf("%dGi", cluster.DindCfg.Storage.StorageSizeInGiB) + storageQuantity, err := resource.ParseQuantity(storageSize) + if err != nil { + return fmt.Errorf("failed to parse quantity %q: %s", storageSize, err) + } + + dindSts.Spec.VolumeClaimTemplates = append(dindSts.Spec.VolumeClaimTemplates, corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: types.DindMountName, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + StorageClassName: util.GetStrPointer(cluster.DindCfg.Storage.StorageClass), + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: storageQuantity, + }, + }, + }, + }) + } + } + } + + err = kclient.Update(ctx, dindSts) + if apierrors.IsInvalid(err) { + deleteOption := metav1.DeletePropagationForeground + err = kclient.Delete(ctx, dindSts, &client.DeleteOptions{ + PropagationPolicy: &deleteOption, + }) + if err != nil { + return fmt.Errorf("failed to delete StatefulSet `dind`: %s", err) + } + + newDindSts := &appsv1.StatefulSet{} + newDindSts.Name = dindSts.Name + newDindSts.Namespace = dindSts.Namespace + newDindSts.Labels = dindSts.Labels + newDindSts.Annotations = dindSts.Annotations + newDindSts.Spec = dindSts.Spec + + return wait.PollImmediate(3*time.Second, 30*time.Second, func() (bool, error) { + err := kclient.Create(ctx, newDindSts) + if err == nil { + return true, nil + } + + log.Warnf("Failed to create StatefulSet `dind`: %s", err) + return false, nil + }) + } + + return err } diff --git a/pkg/microservice/aslan/core/multicluster/service/constants.go b/pkg/microservice/aslan/core/multicluster/service/constants.go new file mode 100644 index 0000000000000000000000000000000000000000..74a2cd458513c83e10cc450f156fc79c84395add --- /dev/null +++ b/pkg/microservice/aslan/core/multicluster/service/constants.go @@ -0,0 +1,67 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "strings" +) + +// StorageProvisioner is a storage type +type StorageProvisioner string + +const ( + // AliCloudNASProvisioner is the provisioner of NFS storage in AliCloud. + AliCloudNASProvisioner StorageProvisioner = "nasplugin.csi.alibabacloud.com" + + // TencentCloudCFSProvisioner is the provisioner of NFS storage in TencentCloud. + TencentCloudCFSProvisioner StorageProvisioner = "com.tencent.cloud.csi.cfs" + + // HuaweiCloudSFSProvisioner is the provisioner of SFS storage in HuaweiCloud. + HuaweiCloudSFSProvisioner StorageProvisioner = "sfsturbo.csi.everest.io" + + // HuaweiCloudNASProvisioner is the provisioner of NAS storage in HuaweiCloud. + HuaweiCloudNASProvisioner StorageProvisioner = "nas.csi.everest.io" + + // AWSEFSProvisioner is the provisioner of EFS storage in AWS. + AWSEFSProvisioner StorageProvisioner = "efs.csi.aws.com" +) + +// Note: For the convenience of users, we filter the storage types for known cloud vendors to prevent users from mistakenly selecting storage +// types that do not support the ``ReadWriteMany`` policy. +// If the user builds their own storage, it is not checked because there is no clear information to judge. +func (s StorageProvisioner) IsNFS() bool { + if strings.Contains(string(s), "alibabacloud.com") && s != AliCloudNASProvisioner { + return false + } + if strings.Contains(string(s), "com.tencent.cloud") && s != TencentCloudCFSProvisioner { + return false + } + if strings.Contains(string(s), "everest.io") && !(s == HuaweiCloudSFSProvisioner || s == HuaweiCloudNASProvisioner) { + return false + } + if strings.Contains(string(s), "aws.com") && s != AWSEFSProvisioner { + return false + } + + return true +} + +// ZadigMinioSVC is the service name of minio. +const ZadigMinioSVC = "zadig-minio" + +// IstiodDeployment is the Deployment name of istiod. +const IstiodDeployment = "istiod" diff --git a/pkg/microservice/aslan/core/multicluster/service/share_env.go b/pkg/microservice/aslan/core/multicluster/service/share_env.go new file mode 100644 index 0000000000000000000000000000000000000000..2ca6820a59445c8e05185d7b874cddd0c6eae62f --- /dev/null +++ b/pkg/microservice/aslan/core/multicluster/service/share_env.go @@ -0,0 +1,60 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "context" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" +) + +func CheckIstiod(ctx context.Context, clusterID string) (bool, error) { + kclient, err := kubeclient.GetKubeClient(config.HubServerAddress(), clusterID) + if err != nil { + return false, fmt.Errorf("failed to get kube client: %s", err) + } + + namespaces := &corev1.NamespaceList{} + err = kclient.List(ctx, namespaces, client.InNamespace("")) + if err != nil { + return false, fmt.Errorf("failed to list namespaces: %s", err) + } + + deployment := &appsv1.Deployment{} + for _, ns := range namespaces.Items { + err = kclient.Get(ctx, client.ObjectKey{Name: IstiodDeployment, Namespace: ns.Name}, deployment) + if err == nil { + return true, nil + } + + if apierrors.IsNotFound(err) { + err = nil + continue + } + + err = fmt.Errorf("failed to search %s: %s", IstiodDeployment, err) + } + + return false, err +} diff --git a/pkg/microservice/aslan/core/project/handler/host.go b/pkg/microservice/aslan/core/project/handler/host.go new file mode 100644 index 0000000000000000000000000000000000000000..5ae80af0e565e3d3f81cfeb818d021b54c4e1830 --- /dev/null +++ b/pkg/microservice/aslan/core/project/handler/host.go @@ -0,0 +1,146 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + + "github.com/gin-gonic/gin" + "github.com/gin-gonic/gin/binding" + + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/system/service" + systemservice "github.com/koderover/zadig/pkg/microservice/aslan/core/system/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/log" +) + +type batchCreatePMArgs struct { + Option string `json:"option"` + Data []*commonmodels.PrivateKey `json:"data"` +} + +func ListPMHosts(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + encryptedKey := c.Query("encryptedKey") + if len(encryptedKey) == 0 { + ctx.Err = e.ErrInvalidParam + return + } + ctx.Resp, ctx.Err = systemservice.ListPrivateKeys(encryptedKey, c.Query("projectName"), c.Query("keyword"), false, ctx.Logger) +} + +func GetPMHost(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.GetPrivateKey(c.Param("id"), ctx.Logger) +} + +func ListLabels(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.ListLabels() +} + +func CreatePMHost(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(commonmodels.PrivateKey) + data, err := c.GetRawData() + if err != nil { + log.Errorf("CreatePMHost c.GetRawData() err : %v", err) + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("CreatePMHost json.Unmarshal err : %v", err) + } + internalhandler.InsertOperationLog(c, ctx.UserName, "", "新增", "项目资源-主机管理", fmt.Sprintf("hostName:%s ip:%s", args.Name, args.IP), string(data), ctx.Logger) + + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + if err := c.ShouldBindWith(&args, binding.JSON); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid physical machine args") + return + } + args.UpdateBy = ctx.UserName + args.ProjectName = c.Query("projectName") + ctx.Err = systemservice.CreatePrivateKey(args, ctx.Logger) +} + +func BatchCreatePMHost(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(batchCreatePMArgs) + data, err := c.GetRawData() + if err != nil { + log.Errorf("BatchCreatePMHost c.GetRawData() err : %v", err) + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("BatchCreatePMHost json.Unmarshal err : %v", err) + } + internalhandler.InsertOperationLog(c, ctx.UserName, "", "批量新增", "项目资源-主机管理", "", string(data), ctx.Logger) + + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + if err := c.ShouldBindJSON(&args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid physical machine args") + return + } + for _, pmArg := range args.Data { + pmArg.ProjectName = c.Query("projectName") + } + ctx.Err = systemservice.BatchCreatePrivateKey(args.Data, args.Option, ctx.UserName, ctx.Logger) +} + +func UpdatePMHost(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(commonmodels.PrivateKey) + data, err := c.GetRawData() + if err != nil { + log.Errorf("UpdatePhysicalHost c.GetRawData() err : %v", err) + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("UpdatePhysicalHost json.Unmarshal err : %v", err) + } + internalhandler.InsertOperationLog(c, ctx.UserName, "", "更新", "项目资源-主机管理", fmt.Sprintf("hostName:%s ip:%s", args.Name, args.IP), string(data), ctx.Logger) + + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + + if err := c.ShouldBindWith(&args, binding.JSON); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid physical machine args") + return + } + args.UpdateBy = ctx.UserName + + ctx.Err = systemservice.UpdatePrivateKey(c.Param("id"), args, ctx.Logger) +} + +func DeletePMHost(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + internalhandler.InsertOperationLog(c, ctx.UserName, "", "删除", "项目资源-主机管理", fmt.Sprintf("id:%s", c.Param("id")), "", ctx.Logger) + ctx.Err = systemservice.DeletePrivateKey(c.Param("id"), ctx.UserName, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/project/handler/product.go b/pkg/microservice/aslan/core/project/handler/product.go index b119ffb71c9eed3681e225746d9e53616ef40812..55ccaab9de5b0316ba86b6f6162c1005d959906e 100644 --- a/pkg/microservice/aslan/core/project/handler/product.go +++ b/pkg/microservice/aslan/core/project/handler/product.go @@ -19,7 +19,9 @@ package handler import ( "bytes" "encoding/json" + "fmt" "io/ioutil" + "strconv" "github.com/gin-gonic/gin" @@ -29,6 +31,7 @@ import ( internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types" ) func GetProductTemplate(c *gin.Context) { @@ -44,20 +47,26 @@ func GetProductTemplateServices(c *gin.Context) { defer func() { internalhandler.JSONResponse(c, ctx) }() productTemplatName := c.Param("name") - ctx.Resp, ctx.Err = projectservice.GetProductTemplateServices(productTemplatName, ctx.Logger) -} -//ListProductTemplate 产品分页信息 -func ListProductTemplate(c *gin.Context) { - ctx := internalhandler.NewContext(c) - defer func() { internalhandler.JSONResponse(c, ctx) }() + envType := types.EnvType(c.Query("envType")) + isBaseEnvStr := c.Query("isBaseEnv") + baseEnvName := c.Query("baseEnv") - productType := c.DefaultQuery("productType", "normal") - if productType != "openSource" { - ctx.Resp, ctx.Err = projectservice.ListProductTemplate(ctx.User.ID, ctx.User.IsSuperUser, ctx.Logger) - return + if envType == "" { + envType = types.GeneralEnv + } + + var isBaseEnv bool + var err error + if envType == types.ShareEnv { + isBaseEnv, err = strconv.ParseBool(isBaseEnvStr) + if err != nil { + ctx.Err = fmt.Errorf("failed to parse %s to bool: %s", isBaseEnvStr, err) + return + } } - ctx.Resp, ctx.Err = projectservice.ListOpenSourceProduct(ctx.Logger) + + ctx.Resp, ctx.Err = projectservice.GetProductTemplateServices(productTemplatName, envType, isBaseEnv, baseEnvName, ctx.Logger) } func CreateProductTemplate(c *gin.Context) { @@ -72,14 +81,14 @@ func CreateProductTemplate(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("CreateProductTemplate json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, args.ProductName, "新增", "项目管理-项目", args.ProductName, string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, args.ProductName, "新增", "项目管理-项目", args.ProductName, string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) if err := c.BindJSON(args); err != nil { ctx.Err = e.ErrInvalidParam.AddDesc("invalid ProductTmpl json args") return } - args.UpdateBy = ctx.Username + args.UpdateBy = ctx.UserName ctx.Err = projectservice.CreateProductTemplate(args, ctx.Logger) } @@ -96,7 +105,7 @@ func UpdateProductTemplate(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("UpdateProductTemplate json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, args.ProductName, "更新", "项目管理-项目环境模板或变量", args.ProductName, string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, args.ProductName, "更新", "项目管理-项目环境模板或变量", args.ProductName, string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) if err := c.BindJSON(args); err != nil { @@ -104,7 +113,7 @@ func UpdateProductTemplate(c *gin.Context) { return } - args.UpdateBy = ctx.Username + args.UpdateBy = ctx.UserName ctx.Err = projectservice.UpdateProductTemplate(c.Param("name"), args, ctx.Logger) } @@ -130,15 +139,15 @@ func UpdateProject(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("UpdateProject json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, args.ProductName, "更新", "项目管理-项目", args.ProductName, string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, args.ProductName, "更新", "项目管理-项目", args.ProductName, string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) if err := c.BindJSON(args); err != nil { ctx.Err = e.ErrInvalidParam.AddDesc("invalid ProductTmpl json args") return } - args.UpdateBy = ctx.Username - productName := c.Query("productName") + args.UpdateBy = ctx.UserName + productName := c.Query("projectName") if productName == "" { ctx.Err = e.ErrInvalidParam.AddDesc("productName can't be empty") return @@ -146,19 +155,50 @@ func UpdateProject(c *gin.Context) { ctx.Err = projectservice.UpdateProject(productName, args, ctx.Logger) } +type UpdateOrchestrationServiceReq struct { + Services [][]string `json:"services"` +} + +func UpdateServiceOrchestration(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + projectName := c.Param("name") + if projectName == "" { + ctx.Err = e.ErrInvalidParam + return + } + internalhandler.InsertOperationLog(c, ctx.UserName, projectName, "更新", "项目管理-项目服务编排", projectName, "", ctx.Logger) + + args := new(UpdateOrchestrationServiceReq) + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid UpdateOrchestrationServiceReq json args") + return + } + + ctx.Err = projectservice.UpdateServiceOrchestration(projectName, args.Services, ctx.UserName, ctx.Logger) +} + func DeleteProductTemplate(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - internalhandler.InsertOperationLog(c, ctx.Username, c.Param("name"), "删除", "项目管理-项目", c.Param("name"), "", ctx.Logger) - ctx.Err = projectservice.DeleteProductTemplate(ctx.Username, c.Param("name"), ctx.RequestID, ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, c.Param("name"), "删除", "项目管理-项目", c.Param("name"), "", ctx.Logger) + isDelete, err := strconv.ParseBool(c.Query("is_delete")) + if err != nil { + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalidParam is_delete") + return + } + } + ctx.Err = projectservice.DeleteProductTemplate(ctx.UserName, c.Param("name"), ctx.RequestID, isDelete, ctx.Logger) } func ListTemplatesHierachy(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = projectservice.ListTemplatesHierachy(ctx.User.Name, ctx.User.ID, ctx.User.IsSuperUser, ctx.Logger) + ctx.Resp, ctx.Err = projectservice.ListTemplatesHierachy(ctx.UserName, ctx.Logger) } func ForkProduct(c *gin.Context) { @@ -171,14 +211,14 @@ func ForkProduct(c *gin.Context) { return } args.ProductName = c.Param("productName") - ctx.Err = projectservice.ForkProduct(ctx.User.ID, ctx.Username, ctx.RequestID, args, ctx.Logger) + ctx.Err = projectservice.ForkProduct(ctx.UserName, ctx.UserID, ctx.RequestID, args, ctx.Logger) } func UnForkProduct(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Err = projectservice.UnForkProduct(ctx.User.ID, ctx.Username, c.Param("productName"), c.Query("workflowName"), c.Query("envName"), ctx.RequestID, ctx.Logger) + ctx.Err = projectservice.UnForkProduct(ctx.UserID, ctx.UserName, c.Param("productName"), c.Query("workflowName"), c.Query("envName"), ctx.RequestID, ctx.Logger) } func GetCustomMatchRules(c *gin.Context) { @@ -202,7 +242,7 @@ func CreateOrUpdateMatchRules(c *gin.Context) { return } - internalhandler.InsertOperationLog(c, ctx.Username, c.Param("name"), "更新", "工程管理-项目", c.Param("name"), "", ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, c.Param("name"), "更新", "工程管理-项目", c.Param("name"), "", ctx.Logger) args := new(projectservice.CustomParseDataArgs) data, err := c.GetRawData() @@ -217,5 +257,5 @@ func CreateOrUpdateMatchRules(c *gin.Context) { return } - ctx.Err = projectservice.UpdateCustomMatchRules(c.Param("name"), ctx.Username, args.Rules) + ctx.Err = projectservice.UpdateCustomMatchRules(c.Param("name"), ctx.UserName, ctx.RequestID, args.Rules) } diff --git a/pkg/microservice/aslan/core/project/handler/project.go b/pkg/microservice/aslan/core/project/handler/project.go index 32bb3949e9978a42e4dc59296479f5c33b705a2e..c2747c35d7a849ffdce97c49ea9e04272fd82814 100644 --- a/pkg/microservice/aslan/core/project/handler/project.go +++ b/pkg/microservice/aslan/core/project/handler/project.go @@ -25,8 +25,10 @@ import ( ) type projectListArgs struct { - IgnoreNoEnvs bool `json:"ignoreNoEnvs" form:"ignoreNoEnvs"` - Verbosity string `json:"verbosity" form:"verbosity"` + IgnoreNoEnvs bool `json:"ignoreNoEnvs" form:"ignoreNoEnvs"` + IgnoreNoVersions bool `json:"ignoreNoVersions" form:"ignoreNoVersions"` + Verbosity string `json:"verbosity" form:"verbosity"` + Names []string `json:"names" form:"names"` } func ListProjects(c *gin.Context) { @@ -39,8 +41,19 @@ func ListProjects(c *gin.Context) { return } + productType := c.DefaultQuery("productType", "normal") + if productType == "openSource" { + ctx.Resp, ctx.Err = projectservice.ListOpenSourceProduct(ctx.Logger) + return + } + ctx.Resp, ctx.Err = projectservice.ListProjects( - &projectservice.ProjectListOptions{IgnoreNoEnvs: args.IgnoreNoEnvs, Verbosity: projectservice.QueryVerbosity(args.Verbosity)}, + &projectservice.ProjectListOptions{ + IgnoreNoEnvs: args.IgnoreNoEnvs, + IgnoreNoVersions: args.IgnoreNoVersions, + Verbosity: projectservice.QueryVerbosity(args.Verbosity), + Names: args.Names, + }, ctx.Logger, ) } diff --git a/pkg/microservice/aslan/core/project/handler/render.go b/pkg/microservice/aslan/core/project/handler/render.go index 86f259fd96deff534d4258460d2a76a41d985ab8..0061ae33955e627aa73d137b182532d7223cf78b 100644 --- a/pkg/microservice/aslan/core/project/handler/render.go +++ b/pkg/microservice/aslan/core/project/handler/render.go @@ -60,31 +60,6 @@ func GetRenderSetInfo(c *gin.Context) { ctx.Resp, ctx.Err = service.GetRenderSetInfo(c.Param("name"), revision) } -//func ValidateRenderSet(c *gin.Context) { -// ctx := internalhandler.NewContext(c) -// defer func() { internalhandler.JSONResponse(c, ctx) }() -// _, ctx.Err = projectservice.ValidateRenderSet(c.Param("productName"), c.Param("renderName"), "", ctx.Logger) -//} -// -//func RelateRender(c *gin.Context) { -// ctx := internalhandler.NewContext(c) -// defer func() { internalhandler.JSONResponse(c, ctx) }() -// ctx.Err = projectservice.RelateRender(c.Param("productName"), c.Param("renderName"), ctx.Logger) -//} -// -//func CreateRenderSet(c *gin.Context) { -// ctx := internalhandler.NewContext(c) -// defer func() { internalhandler.JSONResponse(c, ctx) }() -// -// args := new(models.RenderSet) -// ctx.Err = c.BindJSON(args) -// if ctx.Err != nil { -// return -// } -// args.UpdateBy = ctx.Username -// ctx.Err = projectservice.CreateRenderSet(args, ctx.Logger) -//} - func UpdateRenderSet(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() @@ -94,6 +69,6 @@ func UpdateRenderSet(c *gin.Context) { if ctx.Err != nil { return } - args.UpdateBy = ctx.Username + args.UpdateBy = ctx.UserName ctx.Err = service.UpdateRenderSet(args, ctx.Logger) } diff --git a/pkg/microservice/aslan/core/project/handler/router.go b/pkg/microservice/aslan/core/project/handler/router.go index 8a50316263fab2331b90a22babf49dcb393729df..793a02c34446ac5ef09f1138b7465d451a993fa2 100644 --- a/pkg/microservice/aslan/core/project/handler/router.go +++ b/pkg/microservice/aslan/core/project/handler/router.go @@ -18,16 +18,11 @@ package handler import ( "github.com/gin-gonic/gin" - - gin2 "github.com/koderover/zadig/pkg/middleware/gin" - "github.com/koderover/zadig/pkg/types/permission" ) type Router struct{} func (*Router) Inject(router *gin.RouterGroup) { - router.Use(gin2.Auth()) - // 查看自定义变量是否被引用 render := router.Group("renders") { @@ -42,14 +37,14 @@ func (*Router) Inject(router *gin.RouterGroup) { { product.GET("/:name", GetProductTemplate) product.GET("/:name/services", GetProductTemplateServices) - product.GET("", ListProductTemplate) product.GET("/:name/searching-rules", GetCustomMatchRules) - product.PUT("/:name/searching-rules", gin2.StoreProductName, gin2.IsHavePermission([]string{permission.SuperUserUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, CreateOrUpdateMatchRules) - product.POST("", gin2.StoreProductName, gin2.IsHavePermission([]string{permission.SuperUserUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, CreateProductTemplate) - product.PUT("/:name", gin2.IsHavePermission([]string{permission.ServiceTemplateEditUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, UpdateProductTemplate) - product.PUT("/:name/:status", gin2.IsHavePermission([]string{permission.ServiceTemplateEditUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, UpdateProductTmplStatus) - product.PUT("", gin2.StoreProductName, gin2.IsHavePermission([]string{permission.SuperUserUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, UpdateProject) - product.DELETE("/:name", gin2.IsHavePermission([]string{permission.SuperUserUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, DeleteProductTemplate) + product.PUT("/:name/searching-rules", CreateOrUpdateMatchRules) + product.POST("", CreateProductTemplate) + product.PUT("/:name", UpdateProductTemplate) + product.PUT("/:name/:status", UpdateProductTmplStatus) + product.PATCH("/:name", UpdateServiceOrchestration) + product.PUT("", UpdateProject) + product.DELETE("/:name", DeleteProductTemplate) } openSource := router.Group("opensource") @@ -67,4 +62,15 @@ func (*Router) Inject(router *gin.RouterGroup) { { project.GET("", ListProjects) } + + pms := router.Group("pms") + { + pms.GET("", ListPMHosts) + pms.GET("/:id", GetPMHost) + pms.GET("/labels", ListLabels) + pms.POST("", CreatePMHost) + pms.POST("/batch", BatchCreatePMHost) + pms.PUT("/:id", UpdatePMHost) + pms.DELETE("/:id", DeletePMHost) + } } diff --git a/pkg/microservice/aslan/core/project/service/product.go b/pkg/microservice/aslan/core/project/service/product.go index 98ad31dcc0bddebfd0bd6e828d0d8884c874d4e6..91558373b861c6fdc286bf0d50020745e319f38e 100644 --- a/pkg/microservice/aslan/core/project/service/product.go +++ b/pkg/microservice/aslan/core/project/service/product.go @@ -17,12 +17,12 @@ limitations under the License. package service import ( + "context" "encoding/json" "fmt" "regexp" "strconv" "strings" - "sync" "github.com/hashicorp/go-multierror" "github.com/pkg/errors" @@ -30,20 +30,27 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/yaml" + configbase "github.com/koderover/zadig/pkg/config" "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/collaboration/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/collaboration/service" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/collaboration" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/collie" environmentservice "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" + service2 "github.com/koderover/zadig/pkg/microservice/aslan/core/label/service" workflowservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/poetry" + "github.com/koderover/zadig/pkg/shared/client/policy" + configclient "github.com/koderover/zadig/pkg/shared/config" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" - "github.com/koderover/zadig/pkg/types/permission" + "github.com/koderover/zadig/pkg/types" + "github.com/koderover/zadig/pkg/util" ) type CustomParseDataArgs struct { @@ -58,7 +65,7 @@ type ImageParseData struct { PresetId int `json:"presetId,omitempty"` } -func GetProductTemplateServices(productName string, log *zap.SugaredLogger) (*template.Product, error) { +func GetProductTemplateServices(productName string, envType types.EnvType, isBaseEnv bool, baseEnvName string, log *zap.SugaredLogger) (*template.Product, error) { resp, err := templaterepo.NewProductColl().Find(productName) if err != nil { log.Errorf("GetProductTemplate error: %v", err) @@ -73,206 +80,13 @@ func GetProductTemplateServices(productName string, log *zap.SugaredLogger) (*te if resp.Services == nil { resp.Services = make([][]string, 0) } - return resp, nil -} - -// ListProductTemplate 列出产品模板分页 -func ListProductTemplate(userID int, superUser bool, log *zap.SugaredLogger) ([]*template.Product, error) { - var ( - err error - errorList = &multierror.Error{} - resp = make([]*template.Product, 0) - tmpls = make([]*template.Product, 0) - productTmpls = make([]*template.Product, 0) - productNameMap = make(map[string][]int64) - productMap = make(map[string]*template.Product) - wg sync.WaitGroup - mu sync.Mutex - maxRoutineNum = 20 // 协程池最大协程数量 - ch = make(chan int, maxRoutineNum) // 控制协程数量 - ) - - poetryCtl := poetry.New(config.PoetryAPIServer(), config.PoetryAPIRootKey()) - - tmpls, err = templaterepo.NewProductColl().List() - if err != nil { - log.Errorf("ProfuctTmpl.List error: %v", err) - return resp, e.ErrListProducts.AddDesc(err.Error()) - } - - for _, product := range tmpls { - if superUser { - product.Role = setting.RoleAdmin - product.PermissionUUIDs = []string{} - product.ShowProject = true - continue - } - productMap[product.ProductName] = product - } - if !superUser { - productNameMap, err = poetryCtl.GetUserProject(userID, log) + if envType == types.ShareEnv && !isBaseEnv { + // At this point the request is from the environment share. + resp.Services, err = environmentservice.GetEnvServiceList(context.TODO(), productName, baseEnvName) if err != nil { - log.Errorf("ProfuctTmpl.List GetUserProject error: %v", err) - return resp, e.ErrListProducts.AddDesc(err.Error()) - } - - // 优先处理客户有明确关联关系的项目 - for productName, roleIDs := range productNameMap { - wg.Add(1) - ch <- 1 - // 临时复制range获取的数据，避免重复操作最后一条数据 - tmpProductName := productName - tmpRoleIDs := roleIDs - - go func(tmpProductName string, tmpRoleIDs []int64) { - defer func() { - <-ch - wg.Done() - }() - - roleID := tmpRoleIDs[0] - product, err := templaterepo.NewProductColl().Find(tmpProductName) - if err != nil { - errorList = multierror.Append(errorList, err) - log.Errorf("ProfuctTmpl.List error: %v", err) - return - } - uuids, err := poetryCtl.GetUserPermissionUUIDs(roleID, tmpProductName, log) - if err != nil { - errorList = multierror.Append(errorList, err) - log.Errorf("ProfuctTmpl.List GetUserPermissionUUIDs error: %v", err) - return - } - if roleID == setting.RoleOwnerID { - product.Role = setting.RoleOwner - product.PermissionUUIDs = []string{} - } else { - product.Role = setting.RoleUser - product.PermissionUUIDs = uuids - } - product.ShowProject = true - mu.Lock() - productTmpls = append(productTmpls, product) - delete(productMap, tmpProductName) - mu.Unlock() - }(tmpProductName, tmpRoleIDs) - } - - wg.Wait() - if errorList.ErrorOrNil() != nil { - return resp, errorList + return nil, fmt.Errorf("failed to get service list from env %s of product %s: %s", baseEnvName, productName, err) } - - // 增加项目里面设置过all-users的权限处理 - for _, product := range productMap { - wg.Add(1) - ch <- 1 - // 临时复制range获取的数据，避免重复操作最后一条数据 - tmpProduct := product - - go func(tmpProduct *template.Product) { - defer func() { - <-ch - wg.Done() - }() - - productRole, _ := poetryCtl.ListRoles(tmpProduct.ProductName, log) - if productRole != nil { - uuids, err := poetryCtl.GetUserPermissionUUIDs(productRole.ID, tmpProduct.ProductName, log) - if err != nil { - errorList = multierror.Append(errorList, err) - log.Errorf("ProfuctTmpl.List GetUserPermissionUUIDs error: %v", err) - return - } - - tmpProduct.Role = setting.RoleUser - tmpProduct.PermissionUUIDs = uuids - tmpProduct.ShowProject = true - mu.Lock() - productTmpls = append(productTmpls, tmpProduct) - delete(productMap, tmpProduct.ProductName) - mu.Unlock() - } - }(tmpProduct) - } - wg.Wait() - if errorList.ErrorOrNil() != nil { - return resp, errorList - } - - // 最后处理剩余的项目 - for _, product := range productMap { - wg.Add(1) - ch <- 1 - // 临时复制range获取的数据，避免重复操作最后一条数据 - tmpProduct := product - - go func(tmpProduct *template.Product) { - defer func() { - <-ch - wg.Done() - }() - - var uuids []string - uuids, err = poetryCtl.GetUserPermissionUUIDs(setting.RoleUserID, "", log) - if err != nil { - errorList = multierror.Append(errorList, err) - log.Errorf("ProfuctTmpl.List GetUserPermissionUUIDs error: %v", err) - return - } - tmpProduct.Role = setting.RoleUser - tmpProduct.PermissionUUIDs = uuids - tmpProduct.ShowProject = false - mu.Lock() - productTmpls = append(productTmpls, tmpProduct) - mu.Unlock() - }(tmpProduct) - } - wg.Wait() - if errorList.ErrorOrNil() != nil { - return resp, errorList - } - // 先清空tmpls中的管理员角色数据后，再插入普通用户角色的数据 - tmpls = make([]*template.Product, 0) - tmpls = append(tmpls, productTmpls...) - } - - err = FillProductTemplateVars(tmpls, log) - if err != nil { - return resp, err - } - - for _, tmpl := range tmpls { - wg.Add(1) - ch <- 1 - - go func(tmpTmpl *template.Product) { - defer func() { - <-ch - wg.Done() - }() - - tmpTmpl.TotalServiceNum, err = commonrepo.NewServiceColl().Count(tmpTmpl.ProductName) - if err != nil { - errorList = multierror.Append(errorList, err) - return - } - - tmpTmpl.TotalEnvNum, err = commonrepo.NewProductColl().Count(tmpTmpl.ProductName) - if err != nil { - errorList = multierror.Append(errorList, err) - return - } - - mu.Lock() - resp = append(resp, tmpTmpl) - mu.Unlock() - }(tmpl) - } - wg.Wait() - if errorList.ErrorOrNil() != nil { - return resp, errorList } return resp, nil @@ -295,7 +109,7 @@ func ListOpenSourceProduct(log *zap.SugaredLogger) ([]*template.Product, error) // CreateProductTemplate 创建产品模板 func CreateProductTemplate(args *template.Product, log *zap.SugaredLogger) (err error) { kvs := args.Vars - // 不保存vas + // do not save vars args.Vars = nil err = commonservice.ValidateKVs(kvs, args.AllServiceInfos(), log) @@ -307,6 +121,21 @@ func CreateProductTemplate(args *template.Product, log *zap.SugaredLogger) (err return e.ErrCreateProduct.AddDesc(err.Error()) } + err = commonrepo.NewProjectClusterRelationColl().Delete(&commonrepo.ProjectClusterRelationOption{ProjectName: args.ProductName}) + if err != nil { + log.Errorf("Failed to delete projectClusterRelation, err:%s", err) + } + for _, clusterID := range args.ClusterIDs { + err = commonrepo.NewProjectClusterRelationColl().Create(&commonmodels.ProjectClusterRelation{ + ProjectName: args.ProductName, + ClusterID: clusterID, + CreatedBy: args.UpdateBy, + }) + if err != nil { + log.Errorf("Failed to create projectClusterRelation, err:%s", err) + } + } + err = templaterepo.NewProductColl().Create(args) if err != nil { log.Errorf("ProductTmpl.Create error: %v", err) @@ -331,6 +160,44 @@ func CreateProductTemplate(args *template.Product, log *zap.SugaredLogger) (err return } +func UpdateServiceOrchestration(name string, services [][]string, updateBy string, log *zap.SugaredLogger) (err error) { + templateProductInfo, err := templaterepo.NewProductColl().Find(name) + if err != nil { + log.Errorf("failed to query productInfo, projectName: %s, err: %s", name, err) + return fmt.Errorf("failed to query productInfo, projectName: %s", name) + } + + //validate services + validServices := sets.NewString() + usedServiceSet := sets.NewString() + for _, serviceList := range templateProductInfo.Services { + validServices.Insert(serviceList...) + } + + for _, serviceSeq := range services { + for _, service := range serviceSeq { + if usedServiceSet.Has(service) { + return fmt.Errorf("duplicated service:%s", service) + } + if !validServices.Has(service) { + return fmt.Errorf("service:%s not in valid service list", service) + } + usedServiceSet.Insert(service) + validServices.Delete(service) + } + } + + if validServices.Len() > 0 { + return fmt.Errorf("service: [%s] not found in params", strings.Join(validServices.List(), ",")) + } + + if err = templaterepo.NewProductColl().UpdateServiceOrchestration(name, services, updateBy); err != nil { + log.Errorf("UpdateChoreographyService error: %v", err) + return e.ErrUpdateProduct.AddErr(err) + } + return nil +} + // UpdateProductTemplate 更新产品模板 func UpdateProductTemplate(name string, args *template.Product, log *zap.SugaredLogger) (err error) { kvs := args.Vars @@ -364,7 +231,7 @@ func UpdateProductTemplate(name string, args *template.Product, log *zap.Sugared } for _, envVars := range args.EnvVars { - //创建集成环境变量 + //创建环境变量 if err = commonservice.CreateRenderSet(&commonmodels.RenderSet{ EnvName: envVars.EnvName, Name: args.ProductName, @@ -407,13 +274,6 @@ func UpdateProject(name string, args *template.Product, log *zap.SugaredLogger) if err != nil { return e.ErrInvalidParam.AddDesc(err.Error()) } - poetryCtl := poetry.New(config.PoetryAPIServer(), config.PoetryAPIRootKey()) - //创建团建和项目之间的关系 - _, err = poetryCtl.AddProductTeam(args.ProductName, args.TeamID, args.UserIDs, log) - if err != nil { - log.Errorf("Project.Create AddProductTeam error: %v", err) - return e.ErrUpdateProduct.AddDesc(err.Error()) - } err = templaterepo.NewProductColl().Update(name, args) if err != nil { @@ -475,7 +335,7 @@ func validateCommonRule(currentRule, ruleType, deliveryType string) error { } currentRule = commonservice.ReplaceRuleVariable(currentRule, &commonservice.Variable{ - "ss", "ss", "ss", "ss", "ss", "ss", "ss", "ss", "ss", + "ss", "ss", "ss", "ss", "ss", "ss", "ss", "ss", "ss", "ss", }) switch deliveryType { case config.ImageResourceType: @@ -496,7 +356,7 @@ func validateCommonRule(currentRule, ruleType, deliveryType string) error { } // DeleteProductTemplate 删除产品模板 -func DeleteProductTemplate(userName, productName, requestID string, log *zap.SugaredLogger) (err error) { +func DeleteProductTemplate(userName, productName, requestID string, isDelete bool, log *zap.SugaredLogger) (err error) { publicServices, err := commonrepo.NewServiceColl().ListMaxRevisions(&commonrepo.ServiceListOption{ProductName: productName, Visibility: setting.PublicService}) if err != nil { log.Errorf("pre delete check failed, err: %s", err) @@ -514,72 +374,89 @@ func DeleteProductTemplate(userName, productName, requestID string, log *zap.Sug } } - poetryCtl := poetry.New(config.PoetryAPIServer(), config.PoetryAPIRootKey()) - - //删除项目团队信息 - if err = poetryCtl.DeleteProductTeam(productName, log); err != nil { - log.Errorf("productTeam.Delete error: %v", err) - return e.ErrDeleteProduct - } - envs, _ := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{Name: productName}) for _, env := range envs { if err = commonrepo.NewProductColl().UpdateStatus(env.EnvName, productName, setting.ProductStatusDeleting); err != nil { - log.Errorf("DeleteProductTemplate Update product Status error: %v", err) + log.Errorf("DeleteProductTemplate Update product Status error: %s", err) return e.ErrDeleteProduct } } if err = commonservice.DeleteRenderSet(productName, log); err != nil { - log.Errorf("DeleteProductTemplate DeleteRenderSet err: %v", err) + log.Errorf("DeleteProductTemplate DeleteRenderSet err: %s", err) return err } if err = DeleteTestModules(productName, requestID, log); err != nil { - log.Errorf("DeleteProductTemplate Delete productName %s test err: %v", productName, err) + log.Errorf("DeleteProductTemplate Delete productName %s test err: %s", productName, err) + return err + } + + // delete collaboration_mode and collaboration_instance + if err := DeleteCollabrationMode(productName, userName, log); err != nil { + log.Errorf("DeleteCollabrationMode err:%s", err) + return err + } + + if err = DeletePolicy(productName, log); err != nil { + log.Errorf("DeletePolicy productName %s err: %s", productName, err) + return err + } + + if err = DeleteLabels(productName, log); err != nil { + log.Errorf("DeleteLabels productName %s err: %s", productName, err) return err } if err = commonservice.DeleteWorkflows(productName, requestID, log); err != nil { - log.Errorf("DeleteProductTemplate Delete productName %s workflow err: %v", productName, err) + log.Errorf("DeleteProductTemplate Delete productName %s workflow err: %s", productName, err) + return err + } + + if err = commonservice.DeleteWorkflowV3s(productName, requestID, log); err != nil { + log.Errorf("DeleteProductTemplate Delete productName %s workflowV3 err: %s", productName, err) + return err + } + + if err = commonservice.DeleteWorkflowV4sByProjectName(productName, log); err != nil { + log.Errorf("DeleteProductTemplate Delete productName %s workflowV4 err: %s", productName, err) return err } if err = commonservice.DeletePipelines(productName, requestID, log); err != nil { - log.Errorf("DeleteProductTemplate Delete productName %s pipeline err: %v", productName, err) + log.Errorf("DeleteProductTemplate Delete productName %s pipeline err: %s", productName, err) return err } - //删除自由编排工作流 - features, err := commonservice.GetFeatures(log) - if err != nil { - log.Errorf("DeleteProductTemplate productName %s getFeatures err: %v", productName, err) + // delete projectClusterRelation + if err = commonrepo.NewProjectClusterRelationColl().Delete(&commonrepo.ProjectClusterRelationOption{ProjectName: productName}); err != nil { + log.Errorf("DeleteProductTemplate Delete productName %s ProjectClusterRelation err: %s", productName, err) } - if strings.Contains(features, string(config.FreestyleType)) { - collieClient := collie.New(config.CollieAPIAddress(), config.PoetryAPIRootKey()) + + // Delete freestyle workflow + cl := configclient.New(configbase.ConfigServiceAddress()) + if enable, err := cl.CheckFeature(setting.ModernWorkflowType); err == nil && enable { + collieClient := collie.New(config.CollieAPIAddress()) if err = collieClient.DeleteCIPipelines(productName, log); err != nil { - log.Errorf("DeleteProductTemplate Delete productName %s freestyle pipeline err: %v", productName, err) + log.Errorf("DeleteProductTemplate Delete productName %s freestyle pipeline err: %s", productName, err) } } err = templaterepo.NewProductColl().Delete(productName) if err != nil { - log.Errorf("ProductTmpl.Delete error: %v", err) + log.Errorf("ProductTmpl.Delete error: %s", err) return e.ErrDeleteProduct } err = commonrepo.NewCounterColl().Delete(fmt.Sprintf("product:%s", productName)) if err != nil { - log.Errorf("Counter.Delete error: %v", err) + log.Errorf("Counter.Delete error: %s", err) return err } services, _ := commonrepo.NewServiceColl().ListMaxRevisions( &commonrepo.ServiceListOption{ProductName: productName, Type: setting.K8SDeployType}, ) - for _, s := range services { - commonservice.ProcessServiceWebhook(nil, s, s.ServiceName, log) - } //删除交付中心 //删除构建/删除测试/删除服务 @@ -588,45 +465,49 @@ func DeleteProductTemplate(userName, productName, requestID string, log *zap.Sug _ = commonrepo.NewBuildColl().Delete("", productName) _ = commonrepo.NewServiceColl().Delete("", "", productName, "", 0) _ = commonservice.DeleteDeliveryInfos(productName, log) - _ = DeleteProductsAsync(userName, productName, requestID, log) + _ = DeleteProductsAsync(userName, productName, requestID, isDelete, log) + + // delete service webhooks after services are deleted + for _, s := range services { + commonservice.ProcessServiceWebhook(nil, s, s.ServiceName, log) + } }() + // 删除workload - go func() { - workloads, _ := commonrepo.NewWorkLoadsStatColl().FindByProductName(productName) - for _, v := range workloads { - // update workloads - tmp := []commonmodels.Workload{} - for _, vv := range v.Workloads { - if vv.ProductName != productName { - tmp = append(tmp, vv) + if isDelete { + go func() { + workloads, _ := commonrepo.NewWorkLoadsStatColl().FindByProductName(productName) + for _, v := range workloads { + // update workloads + tmp := []commonmodels.Workload{} + for _, vv := range v.Workloads { + if vv.ProductName != productName { + tmp = append(tmp, vv) + } } + v.Workloads = tmp + commonrepo.NewWorkLoadsStatColl().UpdateWorkloads(v) } - v.Workloads = tmp - commonrepo.NewWorkLoadsStatColl().UpdateWorkloads(v) + }() + } + // delete servicesInExternalEnv data + go func() { + _ = commonrepo.NewServicesInExternalEnvColl().Delete(&commonrepo.ServicesInExternalEnvArgs{ + ProductName: productName, + }) + }() + + // delete privateKey data + go func() { + if err = commonrepo.NewPrivateKeyColl().BulkDelete(productName); err != nil { + log.Errorf("failed to bulk delete privateKey, error:%s", err) } }() return nil } -func ForkProduct(userID int, username, requestID string, args *template.ForkProject, log *zap.SugaredLogger) error { - poetryClient := poetry.New(config.PoetryAPIServer(), config.PoetryAPIRootKey()) - // first check if the product have contributor role, if not, create one - if !poetryClient.ContributorRoleExist(args.ProductName, log) { - err := poetryClient.CreateContributorRole(args.ProductName, log) - if err != nil { - log.Errorf("Cannot create contributor role for product: %s, the error is: %v", args.ProductName, err) - return e.ErrForkProduct.AddDesc(err.Error()) - } - } - - // Give contributor role to this user - // first look for roleID - roleID := poetryClient.GetContributorRoleID(args.ProductName, log) - if roleID < 0 { - log.Errorf("Failed to get contributor Role ID from poetry client") - return e.ErrForkProduct.AddDesc("Failed to get contributor Role ID from poetry client") - } +func ForkProduct(username, uid, requestID string, args *template.ForkProject, log *zap.SugaredLogger) error { prodTmpl, err := templaterepo.NewProductColl().Find(args.ProductName) if err != nil { @@ -668,6 +549,7 @@ func ForkProduct(userID int, username, requestID string, args *template.ForkProj Name: c.Name, Image: c.Image, ImagePath: c.ImagePath, + ImageName: util.GetImageNameFromContainerInfo(c.ImageName, c.Name), } serviceResp.Containers = append(serviceResp.Containers, container) } @@ -699,24 +581,6 @@ func ForkProduct(userID int, username, requestID string, args *template.ForkProj return e.ErrForkProduct.AddDesc(errMsg) } - userList, _ := poetryClient.ListPermissionUsers(args.ProductName, roleID, poetry.ProjectType, log) - newUserList := append(userList, userID) - err = poetryClient.UpdateUserRole(roleID, poetry.ProjectType, args.ProductName, newUserList, log) - if err != nil { - log.Errorf("Failed to update user role, the error is: %v", err) - return e.ErrForkProduct.AddDesc(fmt.Sprintf("Failed to update user role, the error is: %v", err)) - } - - err = poetryClient.CreateUserEnvPermission(&poetry.UserEnvPermission{ - UserID: userID, - ProductName: args.ProductName, - EnvName: args.EnvName, - PermissionUUIDs: []string{permission.TestEnvListUUID, permission.TestEnvManageUUID}, - }) - if err != nil { - return e.ErrForkProduct.AddDesc(fmt.Sprintf("Failed to create env permission for user: %s", username)) - } - workflowPreset, err := workflowservice.PreSetWorkflow(args.ProductName, log) if err != nil { errMsg := fmt.Sprintf("Failed to get workflow preset info, the error is: %+v", err) @@ -757,18 +621,20 @@ func ForkProduct(userID int, username, requestID string, args *template.ForkProj CreateBy: username, UpdateBy: username, } + err = policy.NewDefault().CreateOrUpdateRoleBinding(args.ProductName, &policy.RoleBinding{ + UID: uid, + Role: string(setting.Contributor), + Preset: true, + }) + if err != nil { + log.Errorf("Failed to create or update roleBinding, err: %s", err) + return e.ErrForkProduct + } return workflowservice.CreateWorkflow(workflowArgs, log) } -func UnForkProduct(userID int, username, productName, workflowName, envName, requestID string, log *zap.SugaredLogger) error { - poetryClient := poetry.New(config.PoetryAPIServer(), config.PoetryAPIRootKey()) - if userEnvPermissions, _ := poetryClient.ListUserEnvPermission(productName, userID, log); len(userEnvPermissions) > 0 { - if err := poetryClient.DeleteUserEnvPermission(productName, username, userID, log); err != nil { - return e.ErrUnForkProduct.AddDesc(fmt.Sprintf("Failed to delete env permission for userID: %d, env: %s, productName: %s, the error is: %+v", userID, username, productName, err)) - } - } - +func UnForkProduct(userID string, username, productName, workflowName, envName, requestID string, log *zap.SugaredLogger) error { if _, err := workflowservice.FindWorkflow(workflowName, log); err == nil { err = commonservice.DeleteWorkflow(workflowName, requestID, false, log) if err != nil { @@ -777,15 +643,13 @@ func UnForkProduct(userID int, username, productName, workflowName, envName, req } } - if roleID := poetryClient.GetContributorRoleID(productName, log); roleID > 0 { - err := poetryClient.DeleteUserRole(roleID, poetry.ProjectType, userID, productName, log) - if err != nil { - log.Errorf("Failed to Delete user from role candidate, the error is: %v", err) - return e.ErrUnForkProduct.AddDesc(err.Error()) - } + policyClient := policy.NewDefault() + err := policyClient.DeleteRoleBinding(configbase.RoleBindingNameFromUIDAndRole(userID, setting.Contributor, ""), productName) + if err != nil { + log.Errorf("Failed to delete roleBinding, err: %s", err) + return e.ErrForkProduct } - - if err := commonservice.DeleteProduct(username, envName, productName, requestID, log); err != nil { + if err := environmentservice.DeleteProduct(username, envName, productName, requestID, true, log); err != nil { _, messageMap := e.ErrorMessage(err) if description, ok := messageMap["description"]; ok { if description != "not found" { @@ -871,14 +735,14 @@ func ensureProductTmpl(args *template.Product) error { return nil } -func DeleteProductsAsync(userName, productName, requestID string, log *zap.SugaredLogger) error { +func DeleteProductsAsync(userName, productName, requestID string, isDelete bool, log *zap.SugaredLogger) error { envs, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{Name: productName}) if err != nil { return e.ErrListProducts.AddDesc(err.Error()) } errList := new(multierror.Error) for _, env := range envs { - err = commonservice.DeleteProduct(userName, env.EnvName, productName, requestID, log) + err = environmentservice.DeleteProduct(userName, env.EnvName, productName, requestID, isDelete, log) if err != nil { errList = multierror.Append(errList, err) } @@ -908,33 +772,17 @@ type ContainerInfo struct { Label string `bson:"label" json:"label"` } -func ListTemplatesHierachy(userName string, userID int, superUser bool, log *zap.SugaredLogger) ([]*ProductInfo, error) { +func ListTemplatesHierachy(userName string, log *zap.SugaredLogger) ([]*ProductInfo, error) { var ( err error resp = make([]*ProductInfo, 0) productTmpls = make([]*template.Product, 0) ) - if superUser { - productTmpls, err = templaterepo.NewProductColl().List() - if err != nil { - log.Errorf("[%s] ProductTmpl.List error: %v", userName, err) - return nil, e.ErrListProducts.AddDesc(err.Error()) - } - } else { - productNameMap, err := poetry.New(config.PoetryAPIServer(), config.PoetryAPIRootKey()).GetUserProject(userID, log) - if err != nil { - log.Errorf("ProfuctTmpl.List GetUserProject error: %v", err) - return resp, e.ErrListProducts.AddDesc(err.Error()) - } - for productName := range productNameMap { - product, err := templaterepo.NewProductColl().Find(productName) - if err != nil { - log.Errorf("ProfuctTmpl.List error: %v", err) - return resp, e.ErrListProducts.AddDesc(err.Error()) - } - productTmpls = append(productTmpls, product) - } + productTmpls, err = templaterepo.NewProductColl().List() + if err != nil { + log.Errorf("[%s] ProductTmpl.List error: %v", userName, err) + return nil, e.ErrListProducts.AddDesc(err.Error()) } for _, productTmpl := range productTmpls { @@ -983,7 +831,7 @@ func GetCustomMatchRules(productName string, log *zap.SugaredLogger) ([]*ImagePa return ret, nil } -func UpdateCustomMatchRules(productName string, userName string, matchRules []*ImageParseData) error { +func UpdateCustomMatchRules(productName, userName, requestID string, matchRules []*ImageParseData) error { productInfo, err := templaterepo.NewProductColl().Find(productName) if err != nil { log.Errorf("query product:%s fail, err:%s", productName, err.Error()) @@ -1025,7 +873,7 @@ func UpdateCustomMatchRules(productName string, userName string, matchRules []*I if err != nil { return err } - err = reParseServices(userName, services, imageRulesToSave) + err = reParseServices(userName, requestID, services, imageRulesToSave) if err != nil { return err } @@ -1040,15 +888,17 @@ func UpdateCustomMatchRules(productName string, userName string, matchRules []*I } // reparse values.yaml for each service -func reParseServices(userName string, serviceList []*commonmodels.Service, matchRules []*template.ImageSearchingRule) error { +func reParseServices(userName, requestID string, serviceList []*commonmodels.Service, matchRules []*template.ImageSearchingRule) error { updatedServiceTmpls := make([]*commonmodels.Service, 0) var err error + var projectName string for _, serviceTmpl := range serviceList { if serviceTmpl.Type != setting.HelmDeployType || serviceTmpl.HelmChart == nil { continue } valuesYaml := serviceTmpl.HelmChart.ValuesYaml + projectName = serviceTmpl.ProductName valuesMap := make(map[string]interface{}) err = yaml.Unmarshal([]byte(valuesYaml), &valuesMap) @@ -1099,5 +949,46 @@ func reParseServices(userName string, serviceList []*commonmodels.Service, match return err } + return environmentservice.AutoDeployHelmServiceToEnvs(userName, requestID, projectName, updatedServiceTmpls, log.SugaredLogger()) +} + +func DeleteCollabrationMode(productName string, userName string, log *zap.SugaredLogger) error { + // find all collaboration mode in this project + res, err := collaboration.GetCollaborationModes([]string{productName}, log) + if err != nil { + log.Errorf("GetCollaborationModes err: %s", err) + return err + } + // delete all collaborationMode + for _, mode := range res.Collaborations { + if err := service.DeleteCollaborationMode(userName, productName, mode.Name, log); err != nil { + log.Errorf("DeleteCollaborationMode err: %s", err) + return err + } + } + // delete all collaborationIns + if err := mongodb.NewCollaborationInstanceColl().DeleteByProject(productName); err != nil { + log.Errorf("fail to DeleteByProject err:%s", err) + return err + } + return nil +} + +func DeletePolicy(productName string, log *zap.SugaredLogger) error { + policy.NewDefault() + if err := policy.NewDefault().DeletePolicies(productName, policy.DeletePoliciesArgs{ + Names: []string{}, + }); err != nil { + log.Errorf("DeletePolicies err :%s", err) + return err + } + return nil +} + +func DeleteLabels(productName string, log *zap.SugaredLogger) error { + if err := service2.DeleteLabelsAndBindingsByProject(productName, log); err != nil { + log.Errorf("delete labels and bindings by project fail , err :%s", err) + return err + } return nil } diff --git a/pkg/microservice/aslan/core/project/service/project.go b/pkg/microservice/aslan/core/project/service/project.go index 7da39d9545237a7b6c3c4d3c36eb7db5e0df6bf4..8a4c72bfbc8ceae0e9afafbaac3aee1a0de77ee2 100644 --- a/pkg/microservice/aslan/core/project/service/project.go +++ b/pkg/microservice/aslan/core/project/service/project.go @@ -17,8 +17,6 @@ limitations under the License. package service import ( - "fmt" - "go.uber.org/zap" "k8s.io/apimachinery/pkg/util/sets" @@ -35,12 +33,21 @@ const ( ) type ProjectListOptions struct { - IgnoreNoEnvs bool - Verbosity QueryVerbosity + IgnoreNoEnvs bool + IgnoreNoVersions bool + Verbosity QueryVerbosity + Names []string } type ProjectDetailedRepresentation struct { *ProjectBriefRepresentation + Alias string `json:"alias"` + Desc string `json:"desc"` + UpdatedAt int64 `json:"updatedAt"` + UpdatedBy string `json:"updatedBy"` + Onboard bool `json:"onboard"` + Public bool `json:"public"` + DeployType string `json:"deployType"` } type ProjectBriefRepresentation struct { @@ -55,7 +62,7 @@ type ProjectMinimalRepresentation struct { func ListProjects(opts *ProjectListOptions, logger *zap.SugaredLogger) (interface{}, error) { switch opts.Verbosity { case VerbosityDetailed: - return nil, fmt.Errorf("not Implemented") + return listDetailedProjectInfos(opts, logger) case VerbosityBrief: return listBriefProjectInfos(opts, logger) case VerbosityMinimal: @@ -65,47 +72,98 @@ func ListProjects(opts *ProjectListOptions, logger *zap.SugaredLogger) (interfac } } +func listDetailedProjectInfos(opts *ProjectListOptions, logger *zap.SugaredLogger) ([]*ProjectDetailedRepresentation, error) { + var res []*ProjectDetailedRepresentation + + nameSet, nameMap, err := getProjects(opts) + if err != nil { + logger.Errorf("Failed to list projects, err: %s", err) + return nil, err + } + + nameWithEnvSet, nameWithEnvMap, err := getProjectsWithEnvs(opts) + if err != nil { + logger.Errorf("Failed to list projects, err: %s", err) + return nil, err + } + + desiredSet := nameSet + if opts.IgnoreNoEnvs { + desiredSet = nameSet.Intersection(nameWithEnvSet) + } + + for name := range desiredSet { + info := nameMap[name] + var deployType string + if info.CreateEnvType == "external" { + deployType = "external" + } else if info.BasicFacility == "cloud_host" { + deployType = "cloud_host" + } else { + deployType = info.DeployType + } + res = append(res, &ProjectDetailedRepresentation{ + ProjectBriefRepresentation: &ProjectBriefRepresentation{ + ProjectMinimalRepresentation: &ProjectMinimalRepresentation{Name: name}, + Envs: nameWithEnvMap[name], + }, + Alias: info.Alias, + Desc: info.Desc, + UpdatedAt: info.UpdatedAt, + UpdatedBy: info.UpdatedBy, + Onboard: info.OnboardStatus != 0, + Public: info.Public, + DeployType: deployType, + }) + } + + return res, nil +} + func listBriefProjectInfos(opts *ProjectListOptions, logger *zap.SugaredLogger) ([]*ProjectBriefRepresentation, error) { var res []*ProjectBriefRepresentation - names, err := templaterepo.NewProductColl().ListNames() + nameSet, _, err := getProjects(opts) if err != nil { - logger.Errorf("Failed to list project names, err: %s", err) + logger.Errorf("Failed to list projects, err: %s", err) return nil, err } - nameWithEnvs, err := mongodb.NewProductColl().ListProjects() + nameWithEnvSet, nameWithEnvMap, err := getProjectsWithEnvs(opts) if err != nil { logger.Errorf("Failed to list projects, err: %s", err) return nil, err } - nameSet := sets.NewString() - for _, name := range names { - nameSet.Insert(name) + desiredSet := nameSet + if opts.IgnoreNoEnvs { + desiredSet = nameSet.Intersection(nameWithEnvSet) } - nameWithEnvsSet := sets.NewString() - for _, nameWithEnv := range nameWithEnvs { - // nameWithEnvs may contain projects which are already deleted. - if !nameSet.Has(nameWithEnv.ProjectName) { - continue - } + for name := range desiredSet { res = append(res, &ProjectBriefRepresentation{ - ProjectMinimalRepresentation: &ProjectMinimalRepresentation{Name: nameWithEnv.ProjectName}, - Envs: nameWithEnv.Envs, + ProjectMinimalRepresentation: &ProjectMinimalRepresentation{Name: name}, + Envs: nameWithEnvMap[name], }) - nameWithEnvsSet.Insert(nameWithEnv.ProjectName) } - if !opts.IgnoreNoEnvs { - for _, name := range names { - if !nameWithEnvsSet.Has(name) { - res = append(res, &ProjectBriefRepresentation{ - ProjectMinimalRepresentation: &ProjectMinimalRepresentation{Name: name}, - }) - } + return res, nil +} + +func listMinimalProjectInfoForDelivery(_ *ProjectListOptions, nameSet sets.String, logger *zap.SugaredLogger) ([]*ProjectMinimalRepresentation, error) { + var res []*ProjectMinimalRepresentation + namesWithDelivery, err := mongodb.NewDeliveryVersionColl().FindProducts() + if err != nil { + logger.Errorf("Failed to list projects by delivery, err: %s", err) + return nil, err + } + + for _, name := range namesWithDelivery { + // namesWithDelivery may contain projects which are already deleted. + if !nameSet.Has(name) { + continue } + res = append(res, &ProjectMinimalRepresentation{Name: name}) } return res, nil @@ -113,13 +171,17 @@ func listBriefProjectInfos(opts *ProjectListOptions, logger *zap.SugaredLogger) func listMinimalProjectInfos(opts *ProjectListOptions, logger *zap.SugaredLogger) ([]*ProjectMinimalRepresentation, error) { var res []*ProjectMinimalRepresentation - - names, err := templaterepo.NewProductColl().ListNames() + names, err := templaterepo.NewProductColl().ListNames(opts.Names) if err != nil { logger.Errorf("Failed to list project names, err: %s", err) return nil, err } + nameSet := sets.NewString(names...) + if opts.IgnoreNoVersions { + return listMinimalProjectInfoForDelivery(opts, nameSet, logger) + } + if !opts.IgnoreNoEnvs { for _, name := range names { res = append(res, &ProjectMinimalRepresentation{Name: name}) @@ -128,24 +190,51 @@ func listMinimalProjectInfos(opts *ProjectListOptions, logger *zap.SugaredLogger return res, nil } - nameSet := sets.NewString() - for _, name := range names { - nameSet.Insert(name) - } - - nameWithEnvs, err := mongodb.NewProductColl().ListProjects() + nameWithEnvSet, _, err := getProjectsWithEnvs(opts) if err != nil { logger.Errorf("Failed to list projects, err: %s", err) return nil, err } - for _, nameWithEnv := range nameWithEnvs { + for name := range nameWithEnvSet { // nameWithEnvs may contain projects which are already deleted. - if !nameSet.Has(nameWithEnv.ProjectName) { + if !nameSet.Has(name) { continue } - res = append(res, &ProjectMinimalRepresentation{Name: nameWithEnv.ProjectName}) + res = append(res, &ProjectMinimalRepresentation{Name: name}) } return res, nil } + +func getProjectsWithEnvs(opts *ProjectListOptions) (sets.String, map[string][]string, error) { + nameWithEnvs, err := mongodb.NewProductColl().ListProjectsInNames(opts.Names) + if err != nil { + return nil, nil, err + } + + nameSet := sets.NewString() + nameMap := make(map[string][]string) + for _, nameWithEnv := range nameWithEnvs { + nameSet.Insert(nameWithEnv.ProjectName) + nameMap[nameWithEnv.ProjectName] = nameWithEnv.Envs + } + + return nameSet, nameMap, nil +} + +func getProjects(opts *ProjectListOptions) (sets.String, map[string]*templaterepo.ProjectInfo, error) { + res, err := templaterepo.NewProductColl().ListProjectBriefs(opts.Names) + if err != nil { + return nil, nil, err + } + + nameSet := sets.NewString() + nameMap := make(map[string]*templaterepo.ProjectInfo) + for _, r := range res { + nameSet.Insert(r.Name) + nameMap[r.Name] = r + } + + return nameSet, nameMap, nil +} diff --git a/pkg/microservice/aslan/core/service.go b/pkg/microservice/aslan/core/service.go index b6c26bf743bb2f2dbfb65e65e3d23985b03b2134..37cd95841928ec265ae16f9d339b6a021101d33d 100644 --- a/pkg/microservice/aslan/core/service.go +++ b/pkg/microservice/aslan/core/service.go @@ -23,26 +23,40 @@ import ( "time" "github.com/hashicorp/go-multierror" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" commonconfig "github.com/koderover/zadig/pkg/config" "github.com/koderover/zadig/pkg/microservice/aslan/config" + modeMongodb "github.com/koderover/zadig/pkg/microservice/aslan/core/collaboration/repository/mongodb" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/nsq" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/webhook" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/workflowcontroller" environmentservice "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" + labelMongodb "github.com/koderover/zadig/pkg/microservice/aslan/core/label/repository/mongodb" + multiclusterservice "github.com/koderover/zadig/pkg/microservice/aslan/core/multicluster/service" systemrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/system/repository/mongodb" systemservice "github.com/koderover/zadig/pkg/microservice/aslan/core/system/service" workflowservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" "github.com/koderover/zadig/pkg/tool/log" mongotool "github.com/koderover/zadig/pkg/tool/mongo" + "github.com/koderover/zadig/pkg/tool/rsa" + "github.com/koderover/zadig/pkg/types" ) const ( webhookController = iota ) +type policyGetter interface { + Policies() []*types.PolicyMeta +} + type Controller interface { Run(workers int, stopCh <-chan struct{}) } @@ -67,6 +81,33 @@ func StartControllers(stopCh <-chan struct{}) { wg.Wait() } +func initRsaKey() { + client, err := kubeclient.GetKubeClient(commonconfig.HubServerServiceAddress(), setting.LocalClusterID) + if err != nil { + log.DPanic(err) + } + clientset, err := kubeclient.GetKubeClientSet(config.HubServerAddress(), setting.LocalClusterID) + if err != nil { + log.DPanic(err) + } + _, err = clientset.CoreV1().Secrets(commonconfig.Namespace()).Get(context.TODO(), setting.RSASecretName, metav1.GetOptions{}) + + if err != nil { + if apierrors.IsNotFound(err) { + err, publicKey, privateKey := rsa.GetRsaKey() + if err != nil { + log.DPanic(err) + } + err = kube.CreateOrUpdateRSASecret(publicKey, privateKey, client) + if err != nil { + log.DPanic(err) + } + } else { + log.DPanic(err) + } + } +} + func Start(ctx context.Context) { log.Init(&log.Config{ Level: commonconfig.LogLevel(), @@ -78,16 +119,27 @@ func Start(ctx context.Context) { initDatabase() initService() + initDinD() systemservice.SetProxyConfig() workflowservice.InitPipelineController() + // update offical plugins + workflowservice.UpdateOfficalPluginRepository(log.SugaredLogger()) + workflowcontroller.InitWorkflowController() // 如果集群环境所属的项目不存在，则删除此集群环境 environmentservice.CleanProducts() environmentservice.ResetProductsStatus() + //Parse the workload dependencies configMap, PVC, ingress, secret + go environmentservice.StartClusterInformer() + go StartControllers(ctx.Done()) + + go multiclusterservice.ClusterApplyUpgradeAgent() + + initRsaKey() } func Stop(ctx context.Context) { @@ -111,6 +163,13 @@ func initService() { } } +func initDinD() { + err := systemservice.SyncDinDForRegistries() + if err != nil { + log.Fatal(err) + } +} + func initDatabase() { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() @@ -128,6 +187,7 @@ func initDatabase() { template.NewProductColl(), commonrepo.NewBasicImageColl(), commonrepo.NewBuildColl(), + commonrepo.NewCallbackRequestColl(), commonrepo.NewCounterColl(), commonrepo.NewCronjobColl(), commonrepo.NewDeliveryActivityColl(), @@ -160,6 +220,7 @@ func initDatabase() { commonrepo.NewStrategyColl(), commonrepo.NewStatsColl(), commonrepo.NewSubscriptionColl(), + commonrepo.NewSystemSettingColl(), commonrepo.NewTaskColl(), commonrepo.NewTestTaskStatColl(), commonrepo.NewTestingColl(), @@ -168,9 +229,26 @@ func initDatabase() { commonrepo.NewWorkflowColl(), commonrepo.NewWorkflowStatColl(), commonrepo.NewWorkLoadsStatColl(), + commonrepo.NewServicesInExternalEnvColl(), + commonrepo.NewExternalLinkColl(), + commonrepo.NewChartColl(), + commonrepo.NewDockerfileTemplateColl(), + commonrepo.NewProjectClusterRelationColl(), + commonrepo.NewEnvResourceColl(), + commonrepo.NewEnvSvcDependColl(), + commonrepo.NewBuildTemplateColl(), + commonrepo.NewScanningColl(), + commonrepo.NewWorkflowV4Coll(), + commonrepo.NewworkflowTaskv4Coll(), + commonrepo.NewWorkflowQueueColl(), + commonrepo.NewPluginRepoColl(), systemrepo.NewAnnouncementColl(), systemrepo.NewOperationLogColl(), + labelMongodb.NewLabelColl(), + labelMongodb.NewLabelBindingColl(), + modeMongodb.NewCollaborationModeColl(), + modeMongodb.NewCollaborationInstanceColl(), } { wg.Add(1) go func(r indexer) { @@ -186,6 +264,11 @@ func initDatabase() { // 初始化数据 commonrepo.NewInstallColl().InitInstallData(systemservice.InitInstallMap()) commonrepo.NewBasicImageColl().InitBasicImageData(systemservice.InitbasicImageInfos()) + commonrepo.NewSystemSettingColl().InitSystemSettings() + + if err := commonrepo.NewS3StorageColl().InitData(); err != nil { + log.Warnf("Failed to init S3 data: %s", err) + } } type indexer interface { diff --git a/pkg/microservice/aslan/core/service/handler/environment.go b/pkg/microservice/aslan/core/service/handler/environment.go new file mode 100644 index 0000000000000000000000000000000000000000..e76430f12bcaa8d5769f8d52860eec64af272534 --- /dev/null +++ b/pkg/microservice/aslan/core/service/handler/environment.go @@ -0,0 +1,69 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/service/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +func GetDeployableEnvs(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.GetDeployableEnvs(c.Param("name"), c.Query("projectName")) +} + +func GetKubeWorkloads(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.GetKubeWorkloads(c.Query("namespace"), c.Query("cluster_id"), ctx.Logger) +} + +func LoadKubeWorkloadsYaml(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(service.LoadKubeWorkloadsYamlReq) + data, err := c.GetRawData() + if err != nil { + ctx.Logger.Errorf("copyHelmProduct c.GetRawData() err : %s", err) + } else if err = json.Unmarshal(data, &args); err != nil { + ctx.Logger.Errorf("copyHelmProduct json.Unmarshal err : %s", err) + } + if err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + + serviceNames := make([]string, 0) + for _, svc := range args.Services { + serviceNames = append(serviceNames, svc.Name) + } + + internalhandler.InsertOperationLog(c, ctx.UserName, args.ProductName, "新增", "项目管理-服务", fmt.Sprintf("服务名称:%s", strings.Join(serviceNames, ",")), string(data), ctx.Logger) + + ctx.Err = service.LoadKubeWorkloadsYaml(ctx.UserName, args, false, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/service/handler/helm.go b/pkg/microservice/aslan/core/service/handler/helm.go index e2db5031779a90c71c33bc108278121f78b305f6..61cac46a9feff16979de9d4939681470194ef3e9 100644 --- a/pkg/microservice/aslan/core/service/handler/helm.go +++ b/pkg/microservice/aslan/core/service/handler/helm.go @@ -17,6 +17,8 @@ limitations under the License. package handler import ( + "encoding/json" + "fmt" "strconv" "github.com/gin-gonic/gin" @@ -46,44 +48,102 @@ func GetHelmServiceModule(c *gin.Context) { func GetFilePath(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = svcservice.GetFilePath(c.Param("serviceName"), c.Param("productName"), c.Query("dir"), ctx.Logger) + revision := int64(0) + var err error + if len(c.Query("revision")) > 0 { + revision, err = strconv.ParseInt(c.Query("revision"), 10, 64) + } + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid revision number") + return + } + ctx.Resp, ctx.Err = svcservice.GetFilePath(c.Param("serviceName"), c.Param("productName"), revision, c.Query("dir"), ctx.Logger) } func GetFileContent(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = svcservice.GetFileContent(c.Param("serviceName"), c.Param("productName"), c.Query("filePath"), c.Query("fileName"), ctx.Logger) + + param := new(svcservice.GetFileContentParam) + err := c.ShouldBindQuery(param) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + + ctx.Resp, ctx.Err = svcservice.GetFileContent(c.Param("serviceName"), c.Param("productName"), param, ctx.Logger) } func CreateOrUpdateHelmService(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("projectName can't be nil") + return + } + args := new(svcservice.HelmServiceCreationArgs) if err := c.BindJSON(args); err != nil { ctx.Err = e.ErrInvalidParam.AddDesc("invalid HelmService json args") return } - args.CreatedBy = ctx.Username + args.CreatedBy, args.RequestID = ctx.UserName, ctx.RequestID + + bs, _ := json.Marshal(args) + internalhandler.InsertOperationLog(c, ctx.UserName, projectName, "新增", "项目管理-服务", fmt.Sprintf("服务名称:%s", args.Name), string(bs), ctx.Logger) - ctx.Err = svcservice.CreateOrUpdateHelmService(c.Query("productName"), args, ctx.Logger) + ctx.Resp, ctx.Err = svcservice.CreateOrUpdateHelmService(projectName, args, false, ctx.Logger) +} + +func UpdateHelmService(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("projectName can't be nil") + return + } + + args := new(svcservice.HelmServiceCreationArgs) + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid HelmService json args") + return + } + args.CreatedBy, args.RequestID = ctx.UserName, ctx.RequestID + + bs, _ := json.Marshal(args) + internalhandler.InsertOperationLog(c, ctx.UserName, projectName, "更新", "项目管理-服务", fmt.Sprintf("服务名称:%s", args.Name), string(bs), ctx.Logger) + + ctx.Resp, ctx.Err = svcservice.CreateOrUpdateHelmService(projectName, args, true, ctx.Logger) } func CreateOrUpdateBulkHelmServices(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("projectName can't be nil") + return + } + args := new(svcservice.BulkHelmServiceCreationArgs) if err := c.BindJSON(args); err != nil { ctx.Err = e.ErrInvalidParam.AddDesc("invalid HelmService json args") return } - args.CreatedBy = ctx.Username + args.CreatedBy, args.RequestID = ctx.UserName, ctx.RequestID - ctx.Resp, ctx.Err = svcservice.CreateOrUpdateBulkHelmService(c.Query("productName"), args, ctx.Logger) + bs, _ := json.Marshal(args) + internalhandler.InsertOperationLog(c, ctx.UserName, c.Query("projectName"), "新增", "项目管理-服务", "", string(bs), ctx.Logger) + + ctx.Resp, ctx.Err = svcservice.CreateOrUpdateBulkHelmService(projectName, args, false, ctx.Logger) } -func UpdateHelmService(c *gin.Context) { +func EditHelmService(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() @@ -92,8 +152,8 @@ func UpdateHelmService(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid HelmServiceArgs json args") return } - args.CreateBy = ctx.Username + args.CreateBy = ctx.UserName args.ProductName = c.Param("productName") - ctx.Err = svcservice.UpdateHelmService(args, ctx.Logger) + ctx.Err = svcservice.EditHelmService(args, ctx.Logger) } diff --git a/pkg/microservice/aslan/core/service/handler/loader.go b/pkg/microservice/aslan/core/service/handler/loader.go index 7a9e6406adf962c4a468b83ac34cac628b5ccb6b..52dc80a9050fe280bd356aeaac2347266394e30c 100644 --- a/pkg/microservice/aslan/core/service/handler/loader.go +++ b/pkg/microservice/aslan/core/service/handler/loader.go @@ -17,6 +17,7 @@ limitations under the License. package handler import ( + "encoding/json" "strconv" "github.com/gin-gonic/gin" @@ -45,7 +46,7 @@ func PreloadServiceTemplate(c *gin.Context) { return } - branchName := c.Param("branchName") + branchName := c.Query("branchName") path := c.Query("path") isDir := c.Query("isDir") == "true" @@ -74,7 +75,7 @@ func LoadServiceTemplate(c *gin.Context) { return } - branchName := c.Param("branchName") + branchName := c.Query("branchName") args := new(svcservice.LoadServiceReq) if err := c.BindJSON(args); err != nil { @@ -84,8 +85,57 @@ func LoadServiceTemplate(c *gin.Context) { remoteName := c.Query("remoteName") repoOwner := c.Query("repoOwner") + namespace := c.Query("namespace") + if namespace == "" { + namespace = repoOwner + } + + // Note we can't get the service name from handler layer since it parsed from files on git repo + bs, _ := json.Marshal(args) + internalhandler.InsertOperationLog(c, ctx.UserName, args.ProductName, "新增", "项目管理-服务", "", string(bs), ctx.Logger) + + ctx.Err = svcservice.LoadServiceFromCodeHost(ctx.UserName, codehostID, repoOwner, namespace, repoName, repoUUID, branchName, remoteName, args, false, ctx.Logger) +} + +func SyncServiceTemplate(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + codehostIDStr := c.Param("codehostId") + + codehostID, err := strconv.Atoi(codehostIDStr) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("cannot convert codehost id to string") + return + } + + repoName := c.Query("repoName") + repoUUID := c.Query("repoUUID") + if repoName == "" && repoUUID == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("repoName and repoUUID cannot be empty at the same time") + return + } + + branchName := c.Query("branchName") + + args := new(svcservice.LoadServiceReq) + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid LoadServiceReq json args") + return + } + + remoteName := c.Query("remoteName") + repoOwner := c.Query("repoOwner") + namespace := c.Query("namespace") + if namespace == "" { + namespace = repoOwner + } + + // Note we can't get the service name from handler layer since it parsed from files on git repo + bs, _ := json.Marshal(args) + internalhandler.InsertOperationLog(c, ctx.UserName, args.ProductName, "更新", "项目管理-服务", "", string(bs), ctx.Logger) - ctx.Err = svcservice.LoadServiceFromCodeHost(ctx.Username, codehostID, repoOwner, repoName, repoUUID, branchName, remoteName, args, ctx.Logger) + ctx.Err = svcservice.LoadServiceFromCodeHost(ctx.UserName, codehostID, repoOwner, namespace, repoName, repoUUID, branchName, remoteName, args, true, ctx.Logger) } // ValidateServiceUpdate ... @@ -108,7 +158,7 @@ func ValidateServiceUpdate(c *gin.Context) { return } - branchName := c.Param("branchName") + branchName := c.Query("branchName") path := c.Query("path") isDir := c.Query("isDir") == "true" diff --git a/pkg/microservice/aslan/core/service/handler/router.go b/pkg/microservice/aslan/core/service/handler/router.go index 30e6514b3d05c4715a9857a9d9284bcd7e9aa518..60350bad7adbbe0bdb833c6d8d20b53b2fa26480 100644 --- a/pkg/microservice/aslan/core/service/handler/router.go +++ b/pkg/microservice/aslan/core/service/handler/router.go @@ -18,15 +18,11 @@ package handler import ( "github.com/gin-gonic/gin" - gin2 "github.com/koderover/zadig/pkg/middleware/gin" - "github.com/koderover/zadig/pkg/types/permission" ) type Router struct{} func (*Router) Inject(router *gin.RouterGroup) { - router.Use(gin2.Auth()) - harbor := router.Group("harbor") { harbor.GET("/project", ListHarborProjects) @@ -41,9 +37,11 @@ func (*Router) Inject(router *gin.RouterGroup) { helm.GET("/:productName/:serviceName/serviceModule", GetHelmServiceModule) helm.GET("/:productName/:serviceName/filePath", GetFilePath) helm.GET("/:productName/:serviceName/fileContent", GetFileContent) - helm.POST("/services", gin2.IsHavePermission([]string{permission.ServiceTemplateManageUUID}, permission.ParamType), CreateOrUpdateHelmService) - helm.POST("/services/bulk", gin2.IsHavePermission([]string{permission.ServiceTemplateManageUUID}, permission.ParamType), CreateOrUpdateBulkHelmServices) - helm.PUT("/:productName", gin2.IsHavePermission([]string{permission.ServiceTemplateManageUUID}, permission.ParamType), UpdateHelmService) + helm.POST("/services", CreateOrUpdateHelmService) + helm.PUT("/services", UpdateHelmService) + helm.POST("/services/bulk", CreateOrUpdateBulkHelmServices) + helm.PUT("/:productName", EditHelmService) + helm.PUT("/services/releaseNaming", HelmReleaseNaming) } k8s := router.Group("services") @@ -51,11 +49,15 @@ func (*Router) Inject(router *gin.RouterGroup) { k8s.GET("", ListServiceTemplate) k8s.GET("/:name/:type", GetServiceTemplate) k8s.GET("/:name", GetServiceTemplateOption) - k8s.POST("", GetServiceTemplateProductName, gin2.IsHavePermission([]string{permission.ServiceTemplateManageUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, CreateServiceTemplate) - k8s.PUT("", GetServiceTemplateObjectProductName, gin2.IsHavePermission([]string{permission.ServiceTemplateManageUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, UpdateServiceTemplate) + k8s.POST("", GetServiceTemplateProductName, CreateServiceTemplate) + k8s.PUT("", UpdateServiceTemplate) k8s.PUT("/yaml/validator", YamlValidator) - k8s.DELETE("/:name/:type", gin2.IsHavePermission([]string{permission.ServiceTemplateDeleteUUID}, permission.QueryType), gin2.UpdateOperationLogStatus, DeleteServiceTemplate) + k8s.PUT("/:name/yaml/view", YamlViewServiceTemplate) + k8s.DELETE("/:name/:type", DeleteServiceTemplate) k8s.GET("/:name/:type/ports", ListServicePort) + k8s.GET("/:name/environments/deployable", GetDeployableEnvs) + k8s.GET("/kube/workloads", GetKubeWorkloads) + k8s.POST("/yaml", LoadKubeWorkloadsYaml) } workload := router.Group("workloads") @@ -72,14 +74,22 @@ func (*Router) Inject(router *gin.RouterGroup) { loader := router.Group("loader") { - loader.GET("/preload/:codehostId/:branchName", PreloadServiceTemplate) - loader.POST("/load/:codehostId/:branchName", LoadServiceTemplate) - loader.GET("/validateUpdate/:codehostId/:branchName", ValidateServiceUpdate) + loader.GET("/preload/:codehostId", PreloadServiceTemplate) + loader.POST("/load/:codehostId", LoadServiceTemplate) + loader.PUT("/load/:codehostId", SyncServiceTemplate) + loader.GET("/validateUpdate/:codehostId", ValidateServiceUpdate) } pm := router.Group("pm") { - pm.POST("/:productName", gin2.IsHavePermission([]string{permission.ServiceTemplateManageUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, CreatePMService) - pm.PUT("/:productName", gin2.IsHavePermission([]string{permission.ServiceTemplateManageUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, UpdatePmServiceTemplate) + pm.PUT("/healthCheckUpdate", UpdateServiceHealthCheckStatus) + pm.POST("/:productName", CreatePMService) + pm.PUT("/:productName", UpdatePmServiceTemplate) + } + + template := router.Group("template") + { + template.POST("/load", LoadServiceFromYamlTemplate) + template.POST("/reload", ReloadServiceFromYamlTemplate) } } diff --git a/pkg/microservice/aslan/core/service/handler/service.go b/pkg/microservice/aslan/core/service/handler/service.go index 683f74aa938fac90ebf21ac4af662ad82ba45b1f..8e10426e6a993202504b52a29a8bdfeb5124956e 100644 --- a/pkg/microservice/aslan/core/service/handler/service.go +++ b/pkg/microservice/aslan/core/service/handler/service.go @@ -21,13 +21,14 @@ import ( "encoding/json" "fmt" "io/ioutil" + "net/http" "strconv" "github.com/gin-gonic/gin" + "k8s.io/apimachinery/pkg/util/sets" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" - "github.com/koderover/zadig/pkg/microservice/aslan/core/service/service" svcservice "github.com/koderover/zadig/pkg/microservice/aslan/core/service/service" "github.com/koderover/zadig/pkg/setting" internalhandler "github.com/koderover/zadig/pkg/shared/handler" @@ -39,13 +40,13 @@ func ListServiceTemplate(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = commonservice.ListServiceTemplate(c.Query("productName"), ctx.Logger) + ctx.Resp, ctx.Err = commonservice.ListServiceTemplate(c.Query("projectName"), ctx.Logger) } func ListWorkloadTemplate(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = commonservice.ListWorkloadTemplate(c.Query("productName"), c.Query("env"), ctx.Logger) + ctx.Resp, ctx.Err = commonservice.ListWorkloadTemplate(c.Query("projectName"), c.Query("env"), ctx.Logger) } func GetServiceTemplate(c *gin.Context) { @@ -56,7 +57,7 @@ func GetServiceTemplate(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid revision number") return } - ctx.Resp, ctx.Err = commonservice.GetServiceTemplate(c.Param("name"), c.Param("type"), c.Query("productName"), setting.ProductStatusDeleting, revision, ctx.Logger) + ctx.Resp, ctx.Err = commonservice.GetServiceTemplate(c.Param("name"), c.Param("type"), c.Query("projectName"), setting.ProductStatusDeleting, revision, ctx.Logger) } func GetServiceTemplateOption(c *gin.Context) { @@ -67,7 +68,7 @@ func GetServiceTemplateOption(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid revision number") return } - ctx.Resp, ctx.Err = svcservice.GetServiceTemplateOption(c.Param("name"), c.Query("productName"), revision, ctx.Logger) + ctx.Resp, ctx.Err = svcservice.GetServiceTemplateOption(c.Param("name"), c.Query("projectName"), revision, ctx.Logger) } func CreateServiceTemplate(c *gin.Context) { @@ -82,16 +83,23 @@ func CreateServiceTemplate(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("CreateServiceTemplate json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, args.ProductName, "新增", "项目管理-服务", fmt.Sprintf("服务名称:%s,版本号:%d", args.ServiceName, args.Revision), string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, args.ProductName, "新增", "项目管理-服务", fmt.Sprintf("服务名称:%s", args.ServiceName), string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) if err := c.BindJSON(args); err != nil { ctx.Err = e.ErrInvalidParam.AddDesc("invalid ServiceTmpl json args") return } - args.CreateBy = ctx.Username - ctx.Resp, ctx.Err = svcservice.CreateServiceTemplate(ctx.Username, args, ctx.Logger) + force, err := strconv.ParseBool(c.Query("force")) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("force params error") + return + } + + args.CreateBy = ctx.UserName + + ctx.Resp, ctx.Err = svcservice.CreateServiceTemplate(ctx.UserName, args, force, ctx.Logger) } func UpdateServiceTemplate(c *gin.Context) { @@ -99,19 +107,31 @@ func UpdateServiceTemplate(c *gin.Context) { defer func() { internalhandler.JSONResponse(c, ctx) }() args := new(commonservice.ServiceTmplObject) - data, err := c.GetRawData() - if err != nil { - log.Errorf("UpdateServiceTemplate c.GetRawData() err : %v", err) + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return } - if err = json.Unmarshal(data, args); err != nil { - log.Errorf("UpdateServiceTemplate json.Unmarshal err : %v", err) + if args.Username != "system" { + internalhandler.InsertOperationLog(c, ctx.UserName, args.ProductName, "更新", "项目管理-服务", fmt.Sprintf("服务名称:%s,版本号:%d", args.ServiceName, args.Revision), "", ctx.Logger) + } + args.Username = ctx.UserName + ctx.Err = svcservice.UpdateServiceVisibility(args) +} + +func UpdateServiceHealthCheckStatus(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(commonservice.ServiceTmplObject) + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return } if args.Username != "system" { - internalhandler.InsertOperationLog(c, ctx.Username, args.ProductName, "更新", "项目管理-服务", fmt.Sprintf("服务名称:%s,版本号:%d", args.ServiceName, args.Revision), "", ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, args.ProductName, "更新", "项目管理-服务", fmt.Sprintf("服务名称:%s,版本号:%d", args.ServiceName, args.Revision), "", ctx.Logger) } - c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) - args.Username = ctx.Username - ctx.Err = svcservice.UpdateServiceTemplate(args) + args.Username = ctx.UserName + ctx.Err = svcservice.UpdateServiceHealthCheckStatus(args) } type ValidatorResp struct { @@ -135,13 +155,53 @@ func YamlValidator(c *gin.Context) { ctx.Resp = resp } +func YamlViewServiceTemplate(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(svcservice.YamlViewServiceTemplateReq) + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid yaml args") + return + } + args.ProjectName = c.Query("projectName") + args.ServiceName = c.Param("name") + if args.ProjectName == "" || args.ServiceName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("projectName or serviceName can't be nil") + return + } + ctx.Resp, ctx.Err = svcservice.YamlViewServiceTemplate(args) +} + +func HelmReleaseNaming(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("projectName can't be nil") + return + } + + args := new(svcservice.ReleaseNamingRule) + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid yaml args") + return + } + + bs, _ := json.Marshal(args) + internalhandler.InsertOperationLog(c, ctx.UserName, projectName, "修改", "项目管理-服务", args.ServiceName, string(bs), ctx.Logger) + + ctx.Err = svcservice.UpdateReleaseNamingRule(ctx.UserName, ctx.RequestID, projectName, args, ctx.Logger) +} + func DeleteServiceTemplate(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - internalhandler.InsertOperationLog(c, ctx.Username, c.Query("productName"), "删除", "项目管理-服务", c.Param("name"), "", ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, c.Query("projectName"), "删除", "项目管理-服务", c.Param("name"), "", ctx.Logger) - ctx.Err = svcservice.DeleteServiceTemplate(c.Param("name"), c.Param("type"), c.Query("productName"), c.DefaultQuery("isEnvTemplate", "true"), c.DefaultQuery("visibility", "public"), ctx.Logger) + ctx.Err = svcservice.DeleteServiceTemplate(c.Param("name"), c.Param("type"), c.Query("projectName"), c.DefaultQuery("isEnvTemplate", "true"), c.DefaultQuery("visibility", "public"), ctx.Logger) } func ListServicePort(c *gin.Context) { @@ -152,52 +212,54 @@ func ListServicePort(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid revision number") return } - ctx.Resp, ctx.Err = svcservice.ListServicePort(c.Param("name"), c.Param("type"), c.Query("productName"), setting.ProductStatusDeleting, revision, ctx.Logger) -} - -type K8sWorkloadsArgs struct { - WorkLoads []commonmodels.Workload `bson:"workLoads" json:"workLoads"` - EnvName string `bson:"env_name" json:"env_name"` - ClusterID string `bson:"cluster_id" json:"cluster_id"` - Namespace string `bson:"namespace" json:"namespace"` - ProductName string `bson:"product_name" json:"product_name"` + ctx.Resp, ctx.Err = svcservice.ListServicePort(c.Param("name"), c.Param("type"), c.Query("projectName"), setting.ProductStatusDeleting, revision, ctx.Logger) } func UpdateWorkloads(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - args := new(service.UpdateWorkloadsArgs) + args := new(svcservice.UpdateWorkloadsArgs) err := c.ShouldBindJSON(args) if err != nil { ctx.Err = e.ErrInvalidParam.AddDesc("invalid UpdateWorkloadsArgs") return } - product := c.Query("productName") + product := c.Query("projectName") env := c.Query("env") if product == "" || env == "" { ctx.Err = e.ErrInvalidParam return } - ctx.Err = service.UpdateWorkloads(c, ctx.RequestID, ctx.Username, product, env, *args, ctx.Logger) + allowedEnvs, found := internalhandler.GetResourcesInHeader(c) + if found { + allowedSet := sets.NewString(allowedEnvs...) + if !allowedSet.Has(env) { + c.String(http.StatusForbidden, "not all input envs are allowed, allowed envs are %v", allowedEnvs) + return + } + } + + ctx.Err = svcservice.UpdateWorkloads(c, ctx.RequestID, ctx.UserName, product, env, *args, ctx.Logger) } func CreateK8sWorkloads(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - args := new(K8sWorkloadsArgs) + args := new(svcservice.K8sWorkloadsArgs) err := c.BindJSON(args) if err != nil { ctx.Err = e.ErrInvalidParam.AddDesc("invalid K8sWorkloadsArgs args") return } - ctx.Err = svcservice.CreateK8sWorkLoads(c, ctx.RequestID, ctx.Username, args.ProductName, args.WorkLoads, args.ClusterID, args.Namespace, args.EnvName, ctx.Logger) + + ctx.Err = svcservice.CreateK8sWorkLoads(c, ctx.RequestID, ctx.UserName, args, ctx.Logger) } func ListAvailablePublicServices(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = svcservice.ListAvailablePublicServices(c.Query("productName"), ctx.Logger) + ctx.Resp, ctx.Err = svcservice.ListAvailablePublicServices(c.Query("projectName"), ctx.Logger) } func GetServiceTemplateProductName(c *gin.Context) { @@ -243,7 +305,7 @@ func CreatePMService(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("CreatePMService json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, c.Param("productName"), "新增", "项目管理-物理机部署服务", fmt.Sprintf("服务名称:%s,版本号:%d", args.ServiceTmplObject.ServiceName, args.ServiceTmplObject.Revision), string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, c.Param("productName"), "新增", "项目管理-物理机部署服务", fmt.Sprintf("服务名称:%s,版本号:%d", args.ServiceTmplObject.ServiceName, args.ServiceTmplObject.Revision), string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) if err := c.BindJSON(args); err != nil { @@ -280,7 +342,7 @@ func CreatePMService(c *gin.Context) { } } - ctx.Err = svcservice.CreatePMService(ctx.Username, args, ctx.Logger) + ctx.Err = svcservice.CreatePMService(ctx.UserName, args, ctx.Logger) } func UpdatePmServiceTemplate(c *gin.Context) { @@ -295,7 +357,7 @@ func UpdatePmServiceTemplate(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("UpdatePmServiceTemplate json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, c.Param("productName"), "更新", "项目管理-主机服务", fmt.Sprintf("服务名称:%s,版本号:%d", args.ServiceTmplObject.ServiceName, args.ServiceTmplObject.Revision), "", ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, c.Param("productName"), "更新", "项目管理-主机服务", fmt.Sprintf("服务名称:%s,版本号:%d", args.ServiceTmplObject.ServiceName, args.ServiceTmplObject.Revision), "", ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) for _, heathCheck := range args.ServiceTmplObject.HealthChecks { @@ -322,5 +384,5 @@ func UpdatePmServiceTemplate(c *gin.Context) { } } } - ctx.Err = commonservice.UpdatePmServiceTemplate(ctx.Username, args, ctx.Logger) + ctx.Err = commonservice.UpdatePmServiceTemplate(ctx.UserName, args, ctx.Logger) } diff --git a/pkg/microservice/aslan/core/service/handler/yaml_template.go b/pkg/microservice/aslan/core/service/handler/yaml_template.go new file mode 100644 index 0000000000000000000000000000000000000000..33267b40723b0606eefc39142be2dd900a384778 --- /dev/null +++ b/pkg/microservice/aslan/core/service/handler/yaml_template.go @@ -0,0 +1,60 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "encoding/json" + "fmt" + + "github.com/gin-gonic/gin" + + svcservice "github.com/koderover/zadig/pkg/microservice/aslan/core/service/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" +) + +func LoadServiceFromYamlTemplate(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + req := new(svcservice.LoadServiceFromYamlTemplateReq) + + if err := c.ShouldBindJSON(req); err != nil { + ctx.Err = err + return + } + + bs, _ := json.Marshal(req) + internalhandler.InsertOperationLog(c, ctx.UserName, req.ProjectName, "新增", "项目管理-服务", fmt.Sprintf("服务名称:%s", req.ServiceName), string(bs), ctx.Logger) + + ctx.Err = svcservice.LoadServiceFromYamlTemplate(ctx.UserName, req, false, ctx.Logger) +} + +func ReloadServiceFromYamlTemplate(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + req := new(svcservice.LoadServiceFromYamlTemplateReq) + if err := c.ShouldBindJSON(req); err != nil { + ctx.Err = err + return + } + + bs, _ := json.Marshal(req) + internalhandler.InsertOperationLog(c, ctx.UserName, req.ProjectName, "更新", "项目管理-服务", fmt.Sprintf("服务名称:%s", req.ServiceName), string(bs), ctx.Logger) + + ctx.Err = svcservice.ReloadServiceFromYamlTemplate(ctx.UserName, req, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/service/service/environment.go b/pkg/microservice/aslan/core/service/service/environment.go new file mode 100644 index 0000000000000000000000000000000000000000..1a44ed19bb2e5e8edf6ff512465d0219c1747144 --- /dev/null +++ b/pkg/microservice/aslan/core/service/service/environment.go @@ -0,0 +1,345 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "fmt" + "strings" + + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/kube/getter" + "github.com/koderover/zadig/pkg/util" +) + +// The service can be deployed only in the following situations: +// 1. All general environments are deployable. +// 2. All base environments are deployable. +// 3. If the service has been deployed in the baseline environment, all sub-environments of the baseline environment +// can deploy the service. +// Otherwise, all sub-environments of the baseline environment cannot deploy the service. +func GetDeployableEnvs(svcName, projectName string) ([]string, error) { + // 1. Get all general environments. + envs0, err := getAllGeneralEnvs(projectName) + if err != nil { + return nil, err + } + + // 2. Get all deployable environments in the context of environment sharing.. + envs1, err := getDeployableShareEnvs(svcName, projectName) + if err != nil { + return nil, err + } + + envs0 = append(envs0, envs1...) + + return envs0, nil +} + +type GetKubeWorkloadsResp struct { + WorkloadsMap map[string][]string `json:"workloads_map"` +} + +func GetKubeWorkloads(namespace, clusterID string, log *zap.SugaredLogger) (*GetKubeWorkloadsResp, error) { + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), clusterID) + if err != nil { + log.Errorf("cluster is not connected [%s] err:%s", clusterID, err) + return nil, err + } + + deployments, err := getter.ListDeployments(namespace, nil, kubeClient) + if err != nil { + log.Errorf("GetKubeWorkloads ListDeployments error, error msg:%s", err) + return nil, err + } + workloadsMap := make(map[string][]string) + var deployNames []string + for _, deployment := range deployments { + deployNames = append(deployNames, deployment.Name) + } + workloadsMap["deployment"] = deployNames + configMaps, err := getter.ListConfigMaps(namespace, nil, kubeClient) + if err != nil { + log.Errorf("GetKubeWorkloads ListConfigMaps error, error msg:%s", err) + return nil, err + } + var configMapNames []string + for _, configmap := range configMaps { + configMapNames = append(configMapNames, configmap.Name) + } + workloadsMap["configmap"] = configMapNames + services, err := getter.ListServices(namespace, nil, kubeClient) + if err != nil { + log.Errorf("GetKubeWorkloads ListServices error, error msg:%s", err) + return nil, err + } + var serviceNames []string + for _, service := range services { + serviceNames = append(serviceNames, service.Name) + } + workloadsMap["service"] = serviceNames + ingresses, err := getter.ListIngresses(namespace, kubeClient, true) + if err != nil { + log.Errorf("GetKubeWorkloads ListIngresses error, error msg:%s", err) + return nil, err + } + var ingressNames []string + for _, ingress := range ingresses.Items { + ingressNames = append(ingressNames, ingress.GetName()) + } + workloadsMap["ingress"] = ingressNames + secrets, err := getter.ListSecrets(namespace, kubeClient) + if err != nil { + log.Errorf("GetKubeWorkloads ListSecrets error, error msg:%s", err) + return nil, err + } + var secretNames []string + for _, secret := range secrets { + secretNames = append(secretNames, secret.Name) + } + workloadsMap["secret"] = secretNames + statefulsets, err := getter.ListStatefulSets(namespace, nil, kubeClient) + if err != nil { + log.Errorf("GetKubeWorkloads ListStatefulSets error, error msg:%s", err) + return nil, err + } + var statefulsetNames []string + for _, statefulset := range statefulsets { + statefulsetNames = append(statefulsetNames, statefulset.Name) + } + workloadsMap["statefulset"] = statefulsetNames + pvcs, err := getter.ListPvcs(namespace, nil, kubeClient) + if err != nil { + log.Errorf("GetKubeWorkloads ListPvcs error, error msg:%s", err) + return nil, err + } + var pvcNames []string + for _, pvc := range pvcs { + pvcNames = append(pvcNames, pvc.Name) + } + workloadsMap["pvc"] = pvcNames + return &GetKubeWorkloadsResp{ + WorkloadsMap: workloadsMap, + }, nil +} + +type ServiceWorkloads struct { + Name string `json:"name"` + WorkloadsMap map[string][]string `json:"workloads_map"` +} + +type LoadKubeWorkloadsYamlReq struct { + ProductName string `json:"product_name"` + Visibility string `json:"visibility"` + Type string `json:"type"` + Namespace string `json:"namespace"` + ClusterID string `json:"cluster_id"` + Services []ServiceWorkloads `json:"services"` +} + +type ServiceYaml struct { + Name string `json:"name"` + Yaml string `json:"yaml"` +} + +type GetKubeWorkloadsYamlResp struct { + Services []ServiceYaml `json:"services"` +} + +func LoadKubeWorkloadsYaml(username string, params *LoadKubeWorkloadsYamlReq, force bool, log *zap.SugaredLogger) error { + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), params.ClusterID) + if err != nil { + log.Errorf("cluster is not connected [%s]", params.ClusterID) + return err + } + for _, service := range params.Services { + var yamls []string + for workloadType, workloads := range service.WorkloadsMap { + switch workloadType { + case "configmap": + for _, workload := range workloads { + bs, _, err := getter.GetConfigMapYamlFormat(params.Namespace, workload, kubeClient) + if len(bs) == 0 || err != nil { + log.Errorf("not found yaml %v", err) + return e.ErrGetService.AddDesc(fmt.Sprintf("get deploy/configmap failed err:%s", err)) + } + + yamls = append(yamls, string(bs)) + } + case "deployment": + for _, workload := range workloads { + bs, _, err := getter.GetDeploymentYamlFormat(params.Namespace, workload, kubeClient) + if len(bs) == 0 || err != nil { + log.Errorf("not found yaml %v", err) + return e.ErrGetService.AddDesc(fmt.Sprintf("get deploy/deployment failed err:%s", err)) + } + + yamls = append(yamls, string(bs)) + } + case "service": + for _, workload := range workloads { + bs, _, err := getter.GetServiceYamlFormat(params.Namespace, workload, kubeClient) + if len(bs) == 0 || err != nil { + log.Errorf("not found yaml %v", err) + return e.ErrGetService.AddDesc(fmt.Sprintf("get deploy/service failed err:%s", err)) + } + + yamls = append(yamls, string(bs)) + } + case "secret": + for _, workload := range workloads { + bs, _, err := getter.GetSecretYamlFormat(params.Namespace, workload, kubeClient) + if len(bs) == 0 || err != nil { + log.Errorf("not found yaml %v", err) + return e.ErrGetService.AddDesc(fmt.Sprintf("get deploy/secret failed err:%s", err)) + } + + yamls = append(yamls, string(bs)) + } + case "ingress": + for _, workload := range workloads { + bs, _, err := getter.GetIngressYamlFormat(params.Namespace, workload, kubeClient) + if len(bs) == 0 || err != nil { + log.Errorf("not found yaml %v", err) + return e.ErrGetService.AddDesc(fmt.Sprintf("get deploy/ingress failed err:%s", err)) + } + + yamls = append(yamls, string(bs)) + } + case "statefulset": + for _, workload := range workloads { + bs, _, err := getter.GetStatefulSetYamlFormat(params.Namespace, workload, kubeClient) + if len(bs) == 0 || err != nil { + log.Errorf("not found yaml %v", err) + return fmt.Errorf("get deploy/statefulset failed err:%s", err) + } + yamls = append(yamls, string(bs)) + } + case "pvc": + for _, workload := range workloads { + bs, _, err := getter.GetPVCYamlFormat(params.Namespace, workload, kubeClient) + if len(bs) == 0 || err != nil { + log.Errorf("not found yaml %v", err) + return fmt.Errorf("get deploy/pvc failed err:%s", err) + } + yamls = append(yamls, string(bs)) + } + default: + return fmt.Errorf("do not support workload kind:%s", workloadType) + } + } + yaml := strings.Join(yamls, setting.YamlFileSeperator) + serviceParam := &commonmodels.Service{ + ProductName: params.ProductName, + ServiceName: service.Name, + Visibility: params.Visibility, + Type: params.Type, + Yaml: yaml, + Source: "spock", + } + _, err := CreateServiceTemplate(username, serviceParam, force, log) + if err != nil { + log.Errorf("CreateServiceTemplate error, msg:%s", err) + return err + } + } + + return nil +} + +func getAllGeneralEnvs(projectName string) ([]string, error) { + envs, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{ + Name: projectName, + ShareEnvEnable: util.GetBoolPointer(false), + }) + if err != nil { + return nil, err + } + + envNames := make([]string, len(envs)) + for i, env := range envs { + envNames[i] = env.EnvName + } + + return envNames, nil +} + +func getDeployableShareEnvs(svcName, projectName string) ([]string, error) { + baseEnvs, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{ + Name: projectName, + ShareEnvEnable: util.GetBoolPointer(true), + ShareEnvIsBase: util.GetBoolPointer(true), + }) + if err != nil { + return nil, err + } + + ret := []string{} + for _, baseEnv := range baseEnvs { + ret = append(ret, baseEnv.EnvName) + + if !hasSvcInEnv(svcName, baseEnv) { + continue + } + + subEnvs, err := getSubEnvs(baseEnv.EnvName, projectName) + if err != nil { + return nil, err + } + + ret = append(ret, subEnvs...) + } + + return ret, nil +} + +func getSubEnvs(baseEnvName, projectName string) ([]string, error) { + envs, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{ + Name: projectName, + ShareEnvEnable: util.GetBoolPointer(true), + ShareEnvIsBase: util.GetBoolPointer(false), + ShareEnvBaseEnv: util.GetStrPointer(baseEnvName), + }) + if err != nil { + return nil, err + } + + envNames := make([]string, len(envs)) + for i, env := range envs { + envNames[i] = env.EnvName + } + + return envNames, nil +} + +func hasSvcInEnv(svcName string, env *commonmodels.Product) bool { + for _, svcGroup := range env.Services { + for _, svc := range svcGroup { + if svc.ServiceName == svcName { + return true + } + } + } + + return false +} diff --git a/pkg/microservice/aslan/core/service/service/helm.go b/pkg/microservice/aslan/core/service/service/helm.go index 5cf4e4dd007729d7247e906ba39b500ec3d86c79..d500707ca905feb24494c169d150e7b41079e358 100644 --- a/pkg/microservice/aslan/core/service/service/helm.go +++ b/pkg/microservice/aslan/core/service/service/helm.go @@ -17,55 +17,50 @@ limitations under the License. package service import ( + "bytes" + "encoding/json" "fmt" "io/fs" "os" + "path" "path/filepath" "strings" "sync" + "text/template" "time" "github.com/27149chen/afero" - "github.com/hashicorp/go-multierror" "github.com/otiai10/copy" "github.com/pkg/errors" "go.uber.org/zap" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/yaml" configbase "github.com/koderover/zadig/pkg/config" "github.com/koderover/zadig/pkg/microservice/aslan/config" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" templatemodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" fsservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/fs" - templatestore "github.com/koderover/zadig/pkg/microservice/aslan/core/templatestore/repository/models" - "github.com/koderover/zadig/pkg/microservice/aslan/core/templatestore/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/codehost" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/helmclient" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/types" + "github.com/koderover/zadig/pkg/util" yamlutil "github.com/koderover/zadig/pkg/util/yaml" ) type HelmService struct { - Services []*models.Service `json:"services"` - FileInfos []*types.FileInfo `json:"file_infos"` -} - -type HelmServiceReq struct { - ProductName string `json:"product_name"` - CreateBy string `json:"create_by"` - CodehostID int `json:"codehost_id"` - RepoOwner string `json:"repo_owner"` - RepoName string `json:"repo_name"` - BranchName string `json:"branch_name"` - FilePaths []string `json:"file_paths"` - SrcPath string `json:"src_path"` + ServiceInfos []*commonmodels.Service `json:"service_infos"` + FileInfos []*types.FileInfo `json:"file_infos"` + Services [][]string `json:"services"` } type HelmServiceArgs struct { @@ -82,8 +77,8 @@ type HelmServiceInfo struct { } type HelmServiceModule struct { - ServiceModules []*ServiceModule `json:"service_modules"` - Service *models.Service `json:"service,omitempty"` + ServiceModules []*ServiceModule `json:"service_modules"` + Service *commonmodels.Service `json:"service,omitempty"` } type Chart struct { @@ -96,13 +91,16 @@ type Chart struct { type helmServiceCreationArgs struct { ChartName string ChartVersion string - ValuesYAML string + ServiceRevision int64 + MergedValues string ServiceName string FilePath string ProductName string CreateBy string + RequestID string CodehostID int Owner string + Namespace string Repo string Branch string RepoLink string @@ -110,20 +108,39 @@ type helmServiceCreationArgs struct { HelmTemplateName string ValuePaths []string ValuesYaml string + Variables []*Variable + GiteePath string + GerritRepoName string + GerritBranchName string + GerritRemoteName string + GerritPath string + GerritCodeHostID int + ChartRepoName string + ValuesSource *commonservice.ValuesDataArgs + CreationDetail interface{} + AutoSync bool } type ChartTemplateData struct { TemplateName string - TemplateData *templatestore.Chart + TemplateData *commonmodels.Chart ChartName string ChartVersion string DefaultValuesYAML []byte // content of values.yaml in template } +type GetFileContentParam struct { + FilePath string `json:"filePath" form:"filePath"` + FileName string `json:"fileName" form:"fileName"` + Revision int64 `json:"revision" form:"revision"` + DeliveryVersion bool `json:"deliveryVersion" form:"deliveryVersion"` +} + func ListHelmServices(productName string, log *zap.SugaredLogger) (*HelmService, error) { helmService := &HelmService{ - Services: []*models.Service{}, - FileInfos: []*types.FileInfo{}, + ServiceInfos: []*commonmodels.Service{}, + FileInfos: []*types.FileInfo{}, + Services: [][]string{}, } opt := &commonrepo.ServiceListOption{ @@ -136,21 +153,99 @@ func ListHelmServices(productName string, log *zap.SugaredLogger) (*HelmService, log.Errorf("[helmService.list] err:%v", err) return nil, e.ErrListTemplate.AddErr(err) } - helmService.Services = services + helmService.ServiceInfos = services if len(services) > 0 { - fis, err := loadServiceFileInfos(services[0].ProductName, services[0].ServiceName, "") + fis, err := loadServiceFileInfos(services[0].ProductName, services[0].ServiceName, 0, "") if err != nil { log.Errorf("Failed to load service file info, err: %s", err) return nil, e.ErrListTemplate.AddErr(err) } helmService.FileInfos = fis } + project, err := templaterepo.NewProductColl().Find(productName) + if err != nil { + log.Errorf("Failed to find project info, err: %s", err) + return nil, e.ErrListTemplate.AddErr(err) + } + helmService.Services = project.Services + return helmService, nil } -func GetHelmServiceModule(serviceName, productName string, revision int64, log *zap.SugaredLogger) (*HelmServiceModule, error) { +func getCreateFromChartTemplate(createFrom interface{}) (*models.CreateFromChartTemplate, error) { + bs, err := json.Marshal(createFrom) + if err != nil { + return nil, err + } + ret := &models.CreateFromChartTemplate{} + err = json.Unmarshal(bs, ret) + return ret, err +} + +func fillServiceTemplateVariables(serviceTemplate *models.Service) error { + if serviceTemplate.Source != setting.SourceFromChartTemplate { + return nil + } + creation, err := getCreateFromChartTemplate(serviceTemplate.CreateFrom) + if err != nil { + return fmt.Errorf("failed to get creation detail: %s", err) + } + + templateChart, err := commonrepo.NewChartColl().Get(creation.TemplateName) + if err != nil { + return err + } + variables := make([]*models.Variable, 0) + curValueMap := make(map[string]string) + for _, kv := range creation.Variables { + curValueMap[kv.Key] = kv.Value + } + + for _, v := range templateChart.ChartVariables { + value := v.Value + if cv, ok := curValueMap[v.Key]; ok { + value = cv + } + variables = append(variables, &models.Variable{Key: v.Key, Value: value}) + } + creation.Variables = variables + serviceTemplate.CreateFrom = creation + return nil +} +func fillServiceCreationInfo(serviceTemplate *models.Service) error { + if serviceTemplate.Source != setting.SourceFromChartTemplate { + return nil + } + creation, err := getCreateFromChartTemplate(serviceTemplate.CreateFrom) + if err != nil { + return fmt.Errorf("failed to get creation detail: %s", err) + } + + if creation.YamlData == nil || creation.YamlData.Source != setting.SourceFromGitRepo { + return nil + } + + bs, err := json.Marshal(creation.YamlData.SourceDetail) + if err != nil { + return err + } + cfr := &models.CreateFromRepo{} + err = json.Unmarshal(bs, cfr) + if err != nil { + return err + } + if cfr.GitRepoConfig == nil { + return nil + } + cfr.GitRepoConfig.Namespace = cfr.GitRepoConfig.GetNamespace() + creation.YamlData.SourceDetail = cfr + serviceTemplate.CreateFrom = creation + return nil +} + +func GetHelmServiceModule(serviceName, productName string, revision int64, log *zap.SugaredLogger) (*HelmServiceModule, error) { serviceTemplate, err := commonservice.GetServiceTemplate(serviceName, setting.HelmDeployType, productName, setting.ProductStatusDeleting, revision, log) if err != nil { return nil, err @@ -160,35 +255,68 @@ func GetHelmServiceModule(serviceName, productName string, revision int64, log * for _, container := range serviceTemplate.Containers { serviceModule := new(ServiceModule) serviceModule.Container = container - buildObj, _ := commonrepo.NewBuildColl().Find(&commonrepo.BuildFindOption{ProductName: productName, ServiceName: serviceName, Targets: []string{container.Name}}) - if buildObj != nil { - serviceModule.BuildName = buildObj.Name + + buildObjs, err := commonrepo.NewBuildColl().List(&commonrepo.BuildListOption{ProductName: productName, ServiceName: serviceName, Targets: []string{container.Name}}) + if err != nil { + return nil, err + } + buildNames := sets.NewString() + for _, buildObj := range buildObjs { + buildNames.Insert(buildObj.Name) } + serviceModule.BuildNames = buildNames.List() serviceModules = append(serviceModules, serviceModule) } + err = fillServiceTemplateVariables(serviceTemplate) + if err != nil { + // NOTE source template may be deleted, error should not block the following logic + log.Warnf("failed to fill service template variables for service: %s, err: %s", serviceTemplate.ServiceName, err) + } + err = fillServiceCreationInfo(serviceTemplate) + if err != nil { + // NOTE since the source of yaml can always be selected when reloading, error should not block the following logic + log.Warnf("failed to fill git namespace for yaml source : %s, err: %s", serviceTemplate.ServiceName, err) + } + helmServiceModule.Service = serviceTemplate + serviceTemplate.ReleaseNaming = serviceTemplate.GetReleaseNaming() helmServiceModule.ServiceModules = serviceModules return helmServiceModule, err } -func GetFilePath(serviceName, productName, dir string, _ *zap.SugaredLogger) ([]*types.FileInfo, error) { - return loadServiceFileInfos(productName, serviceName, dir) +func GetFilePath(serviceName, productName string, revision int64, dir string, _ *zap.SugaredLogger) ([]*types.FileInfo, error) { + return loadServiceFileInfos(productName, serviceName, revision, dir) } -func GetFileContent(serviceName, productName, filePath, fileName string, log *zap.SugaredLogger) (string, error) { - base := config.LocalServicePath(productName, serviceName) - +func GetFileContent(serviceName, productName string, param *GetFileContentParam, log *zap.SugaredLogger) (string, error) { + filePath, fileName, revision, forDelivery := param.FilePath, param.FileName, param.Revision, param.DeliveryVersion svc, err := commonrepo.NewServiceColl().Find(&commonrepo.ServiceFindOption{ ProductName: productName, ServiceName: serviceName, + Revision: revision, }) if err != nil { return "", e.ErrFileContent.AddDesc(err.Error()) } - err = commonservice.PreLoadServiceManifests(base, svc) - if err != nil { - return "", e.ErrFileContent.AddDesc(err.Error()) + base := config.LocalServicePath(productName, serviceName) + if revision > 0 { + base = config.LocalServicePathWithRevision(productName, serviceName, revision) + if err = commonservice.PreloadServiceManifestsByRevision(base, svc); err != nil { + log.Warnf("failed to get chart of revision: %d for service: %s, use latest version", + svc.Revision, svc.ServiceName) + } + } + if err != nil || revision == 0 { + base = config.LocalServicePath(productName, serviceName) + err = commonservice.PreLoadServiceManifests(base, svc) + if err != nil { + return "", e.ErrFileContent.AddDesc(err.Error()) + } + } + + if forDelivery { + base = config.LocalDeliveryChartPathWithRevision(productName, serviceName, revision) } file := filepath.Join(base, serviceName, filePath, fileName) @@ -202,16 +330,17 @@ func GetFileContent(serviceName, productName, filePath, fileName string, log *za } func prepareChartTemplateData(templateName string, logger *zap.SugaredLogger) (*ChartTemplateData, error) { - // get chart template from local disk - templateChart, err := mongodb.NewChartColl().Get(templateName) + templateChart, err := commonrepo.NewChartColl().Get(templateName) if err != nil { logger.Errorf("Failed to get chart template %s, err: %s", templateName, err) - return nil, err + return nil, fmt.Errorf("failed to get chart template: %s", templateName) } + // get chart template from local disk localBase := configbase.LocalChartTemplatePath(templateName) s3Base := configbase.ObjectStorageChartTemplatePath(templateName) - if err = fsservice.PreloadFiles(templateName, localBase, s3Base, logger); err != nil { + if err = fsservice.PreloadFiles(templateName, localBase, s3Base, templateChart.Source, logger); err != nil { + logger.Errorf("Failed to download template %s, err: %s", templateName, err) return nil, err } @@ -222,6 +351,7 @@ func prepareChartTemplateData(templateName string, logger *zap.SugaredLogger) (* chartFilePath := filepath.Join(localBase, base, setting.ChartYaml) chartFileContent, err := os.ReadFile(chartFilePath) if err != nil { + logger.Errorf("Failed to read chartfile template %s, err: %s", templateName, err) return nil, err } chart := new(Chart) @@ -239,46 +369,214 @@ func prepareChartTemplateData(templateName string, logger *zap.SugaredLogger) (* }, nil } -func CreateOrUpdateHelmService(projectName string, args *HelmServiceCreationArgs, logger *zap.SugaredLogger) error { +func getNextServiceRevision(productName, serviceName string) (int64, error) { + serviceTemplate := fmt.Sprintf(setting.ServiceTemplateCounterName, serviceName, productName) + rev, err := commonrepo.NewCounterColl().GetNextSeq(serviceTemplate) + if err != nil { + log.Errorf("Failed to get next revision for service %s, err: %s", serviceName, err) + return 0, err + } + if err = commonrepo.NewServiceColl().Delete(serviceName, setting.HelmDeployType, serviceName, setting.ProductStatusDeleting, rev); err != nil { + log.Warnf("Failed to delete stale service %s with revision %d, err: %s", serviceName, rev, err) + } + return rev, err +} + +// make local chart info copy with revision +func copyChartRevision(projectName, serviceName string, revision int64) error { + sourceChartPath := config.LocalServicePath(projectName, serviceName) + revisionChartLocalPath := config.LocalServicePathWithRevision(projectName, serviceName, revision) + + err := os.RemoveAll(revisionChartLocalPath) + if err != nil { + log.Errorf("failed to remove old chart revision data, projectName %s serviceName %s revision %d, err %s", projectName, serviceName, revision, err) + return err + } + + err = copy.Copy(sourceChartPath, revisionChartLocalPath) + if err != nil { + log.Errorf("failed to copy chart info, projectName %s serviceName %s revision %d, err %s", projectName, serviceName, revision, err) + return err + } + return nil +} + +func clearChartFiles(projectName, serviceName string, revision int64, logger *zap.SugaredLogger, source ...string) { + clearChartFilesInS3Storage(projectName, serviceName, revision, logger) + if len(source) == 0 { + clearLocalChartFiles(projectName, serviceName, revision, logger) + } +} + +// clear chart files in s3 storage +func clearChartFilesInS3Storage(projectName, serviceName string, revision int64, logger *zap.SugaredLogger) { + s3FileNames := []string{serviceName, fmt.Sprintf("%s-%d", serviceName, revision)} + errRemoveFile := fsservice.DeleteArchivedFileFromS3(s3FileNames, config.ObjectStorageServicePath(projectName, serviceName), logger) + if errRemoveFile != nil { + logger.Errorf("Failed to remove files: %v from s3 strorage, err: %s", s3FileNames, errRemoveFile) + } +} + +// clear local chart infos +func clearLocalChartFiles(projectName, serviceName string, revision int64, logger *zap.SugaredLogger) { + latestChartPath := config.LocalServicePath(projectName, serviceName) + revisionChartLocalPath := config.LocalServicePathWithRevision(projectName, serviceName, revision) + for _, path := range []string{latestChartPath, revisionChartLocalPath} { + err := os.RemoveAll(path) + if err != nil { + logger.Errorf("failed to remove local chart data, path: %s, err: %s", path, err) + } + } +} + +func CreateOrUpdateHelmService(projectName string, args *HelmServiceCreationArgs, force bool, logger *zap.SugaredLogger) (*BulkHelmServiceCreationResponse, error) { switch args.Source { case LoadFromRepo, LoadFromPublicRepo: - return CreateOrUpdateHelmServiceFromGitRepo(projectName, args, logger) + return CreateOrUpdateHelmServiceFromGitRepo(projectName, args, force, logger) case LoadFromChartTemplate: - return CreateOrUpdateHelmServiceFromChartTemplate(projectName, args, logger) + return CreateOrUpdateHelmServiceFromChartTemplate(projectName, args, force, logger) + case LoadFromGerrit, setting.SourceFromGitee: + return CreateOrUpdateHelmServiceFromRepo(projectName, args, force, logger) + case LoadFromChartRepo: + return CreateOrUpdateHelmServiceFromChartRepo(projectName, args, force, logger) default: - return fmt.Errorf("invalid source") + return nil, fmt.Errorf("invalid source") } } -func CreateOrUpdateHelmServiceFromChartTemplate(projectName string, args *HelmServiceCreationArgs, logger *zap.SugaredLogger) error { +func CreateOrUpdateHelmServiceFromChartRepo(projectName string, args *HelmServiceCreationArgs, force bool, log *zap.SugaredLogger) (*BulkHelmServiceCreationResponse, error) { + chartRepoArgs, ok := args.CreateFrom.(*CreateFromChartRepo) + if !ok { + return nil, e.ErrCreateTemplate.AddDesc("invalid argument") + } + + chartRepo, err := commonrepo.NewHelmRepoColl().Find(&commonrepo.HelmRepoFindOption{RepoName: chartRepoArgs.ChartRepoName}) + if err != nil { + log.Errorf("failed to query chart-repo info, productName: %s, err: %s", projectName, err) + return nil, e.ErrCreateTemplate.AddDesc(fmt.Sprintf("failed to query chart-repo info, productName: %s, repoName: %s", projectName, chartRepoArgs.ChartRepoName)) + } + + hClient, err := helmclient.NewClient() + if err != nil { + return nil, e.ErrCreateTemplate.AddErr(errors.Wrapf(err, "failed to init chart client for repo: %s", chartRepo.RepoName)) + } + + chartRef := fmt.Sprintf("%s/%s", chartRepo.RepoName, chartRepoArgs.ChartName) + localPath := config.LocalServicePath(projectName, chartRepoArgs.ChartName) + // remove local file to untar + _ = os.RemoveAll(localPath) + err = hClient.DownloadChart(commonservice.GeneHelmRepo(chartRepo), chartRef, chartRepoArgs.ChartVersion, localPath, true) + if err != nil { + return nil, e.ErrCreateTemplate.AddErr(errors.Wrapf(err, "failed to download chart %s/%s-%s", chartRepo.RepoName, chartRepoArgs.ChartName, chartRepoArgs.ChartVersion)) + } + + serviceName := chartRepoArgs.ChartName + rev, err := getNextServiceRevision(projectName, serviceName) + if err != nil { + log.Errorf("Failed to get next revision for service %s, err: %s", serviceName, err) + return nil, e.ErrCreateTemplate.AddErr(err) + } + + var finalErr error + // clear files from both s3 and local when error occurred in next stages + defer func() { + if finalErr != nil { + clearChartFiles(projectName, serviceName, rev, log) + } + }() + + // read values.yaml + fsTree := os.DirFS(config.LocalServicePath(projectName, chartRepoArgs.ChartName)) + valuesYAML, err := readValuesYAML(fsTree, chartRepoArgs.ChartName, log) + if err != nil { + finalErr = e.ErrCreateTemplate.AddErr(err) + return nil, finalErr + } + + // upload to s3 storage + s3Base := config.ObjectStorageServicePath(projectName, serviceName) + err = fsservice.ArchiveAndUploadFilesToS3(fsTree, []string{serviceName, fmt.Sprintf("%s-%d", serviceName, rev)}, s3Base, log) + if err != nil { + finalErr = e.ErrCreateTemplate.AddErr(err) + return nil, finalErr + } + + // copy service revision data from latest + err = copyChartRevision(projectName, serviceName, rev) + if err != nil { + log.Errorf("Failed to copy file %s, err: %s", serviceName, err) + finalErr = errors.Wrapf(err, "Failed to copy chart info, service %s", serviceName) + return nil, finalErr + } + + svc, err := createOrUpdateHelmService( + fsTree, + &helmServiceCreationArgs{ + ChartName: chartRepoArgs.ChartName, + ChartVersion: chartRepoArgs.ChartVersion, + ChartRepoName: chartRepoArgs.ChartRepoName, + ServiceRevision: rev, + MergedValues: string(valuesYAML), + ServiceName: serviceName, + ProductName: projectName, + CreateBy: args.CreatedBy, + RequestID: args.RequestID, + Source: setting.SourceFromChartRepo, + }, force, + log, + ) + if err != nil { + log.Errorf("Failed to create service %s in project %s, error: %s", serviceName, projectName, err) + finalErr = e.ErrCreateTemplate.AddErr(err) + return nil, finalErr + } + + compareHelmVariable([]*templatemodels.RenderChart{ + { + ServiceName: chartRepoArgs.ChartName, + ChartVersion: svc.HelmChart.Version, + ValuesYaml: svc.HelmChart.ValuesYaml, + }, + }, projectName, args.CreatedBy, log) + + err = service.AutoDeployHelmServiceToEnvs(args.CreatedBy, args.RequestID, svc.ProductName, []*models.Service{svc}, log) + if err != nil { + finalErr = e.ErrCreateTemplate.AddErr(err) + return nil, finalErr + } + + return &BulkHelmServiceCreationResponse{ + SuccessServices: []string{serviceName}, + }, nil +} + +func CreateOrUpdateHelmServiceFromChartTemplate(projectName string, args *HelmServiceCreationArgs, force bool, logger *zap.SugaredLogger) (*BulkHelmServiceCreationResponse, error) { templateArgs, ok := args.CreateFrom.(*CreateFromChartTemplate) if !ok { - return fmt.Errorf("invalid argument") + return nil, fmt.Errorf("invalid argument") } templateChartInfo, err := prepareChartTemplateData(templateArgs.TemplateName, logger) if err != nil { - return err + return nil, err } + return createOrUpdateHelmServiceFromChartTemplate(templateArgs, templateChartInfo, projectName, args, force, logger) +} + +func createOrUpdateHelmServiceFromChartTemplate(templateArgs *CreateFromChartTemplate, templateChartInfo *ChartTemplateData, projectName string, args *HelmServiceCreationArgs, force bool, logger *zap.SugaredLogger) (*BulkHelmServiceCreationResponse, error) { + + // NOTE we may need a better way to handle service name with spaces + args.Name = strings.TrimSpace(args.Name) + var values [][]byte if len(templateChartInfo.DefaultValuesYAML) > 0 { - values = append(values, templateChartInfo.DefaultValuesYAML) - } - - for _, path := range templateArgs.ValuesPaths { - v, err := fsservice.DownloadFileFromSource(&fsservice.DownloadFromSourceArgs{ - CodehostID: templateArgs.CodehostID, - Owner: templateArgs.Owner, - Repo: templateArgs.Repo, - Path: path, - Branch: templateArgs.Branch, - }) + //render variables + renderedYaml, err := renderVariablesToYaml(string(templateChartInfo.DefaultValuesYAML), projectName, args.Name, templateArgs.Variables) if err != nil { - return err + return nil, err } - - values = append(values, v) + values = append(values, []byte(renderedYaml)) } if len(templateArgs.ValuesYAML) > 0 { @@ -291,139 +589,332 @@ func CreateOrUpdateHelmServiceFromChartTemplate(projectName string, args *HelmSe // copy template to service path and update the values.yaml from := filepath.Join(localBase, base) to := filepath.Join(config.LocalServicePath(projectName, args.Name), args.Name) - if err = copy.Copy(from, to); err != nil { + // remove old files + if err := os.RemoveAll(to); err != nil { + logger.Errorf("Failed to remove dir %s, err: %s", to, err) + return nil, err + } + if err := copy.Copy(from, to); err != nil { logger.Errorf("Failed to copy file from %s to %s, err: %s", from, to, err) - return err + return nil, err } merged, err := yamlutil.Merge(values) if err != nil { logger.Errorf("Failed to merge values, err: %s", err) - return err + return nil, err } if err = os.WriteFile(filepath.Join(to, setting.ValuesYaml), merged, 0644); err != nil { logger.Errorf("Failed to write values, err: %s", err) - return err + return nil, err + } + + rev, err := getNextServiceRevision(projectName, args.Name) + if err != nil { + logger.Errorf("Failed to get next revision for service %s, err: %s", args.Name, err) + return nil, errors.Wrapf(err, "Failed to get service next revision, service %s", args.Name) } + err = copyChartRevision(projectName, args.Name, rev) + if err != nil { + logger.Errorf("Failed to copy file %s, err: %s", args.Name, err) + return nil, errors.Wrapf(err, "Failed to copy chart info, service %s", args.Name) + } + + // clear files from both s3 and local when error occurred in next stages + defer func() { + if err != nil { + clearChartFiles(projectName, args.Name, rev, logger) + } + }() + fsTree := os.DirFS(config.LocalServicePath(projectName, args.Name)) - ServiceS3Base := config.ObjectStorageServicePath(projectName, args.Name) - if err = fsservice.ArchiveAndUploadFilesToS3(fsTree, args.Name, ServiceS3Base, logger); err != nil { + serviceS3Base := config.ObjectStorageServicePath(projectName, args.Name) + if err = fsservice.ArchiveAndUploadFilesToS3(fsTree, []string{args.Name, fmt.Sprintf("%s-%d", args.Name, rev)}, serviceS3Base, logger); err != nil { logger.Errorf("Failed to upload files for service %s in project %s, err: %s", args.Name, projectName, err) - return err + return nil, err } - svc, err := createOrUpdateHelmService( + svc, errCreate := createOrUpdateHelmService( fsTree, &helmServiceCreationArgs{ ChartName: templateChartInfo.ChartName, ChartVersion: templateChartInfo.ChartVersion, - ValuesYAML: string(merged), + ServiceRevision: rev, + MergedValues: string(merged), ServiceName: args.Name, FilePath: to, ProductName: projectName, CreateBy: args.CreatedBy, - CodehostID: templateArgs.CodehostID, - Owner: templateArgs.Owner, - Repo: templateArgs.Repo, - Branch: templateArgs.Branch, + RequestID: args.RequestID, Source: setting.SourceFromChartTemplate, HelmTemplateName: templateArgs.TemplateName, - ValuePaths: templateArgs.ValuesPaths, ValuesYaml: templateArgs.ValuesYAML, - }, + Variables: templateArgs.Variables, + ValuesSource: args.ValuesData, + CreationDetail: args.CreationDetail, + AutoSync: args.AutoSync, + }, force, logger, ) - if err != nil { + + if errCreate != nil { + err = errCreate logger.Errorf("Failed to create service %s in project %s, error: %s", args.Name, projectName, err) - return err + return nil, err } - if svc.HelmChart != nil { - compareHelmVariable([]*templatemodels.RenderChart{ - {ServiceName: args.Name, - ChartVersion: svc.HelmChart.Version, - ValuesYaml: svc.HelmChart.ValuesYaml, - }, - }, projectName, args.CreatedBy, logger) + compareHelmVariable([]*templatemodels.RenderChart{ + { + ServiceName: args.Name, + ChartVersion: svc.HelmChart.Version, + ValuesYaml: svc.HelmChart.ValuesYaml, + }, + }, projectName, args.CreatedBy, logger) + + err = service.AutoDeployHelmServiceToEnvs(args.CreatedBy, args.RequestID, svc.ProductName, []*models.Service{svc}, logger) + if err != nil { + return nil, err } - return nil + return &BulkHelmServiceCreationResponse{ + SuccessServices: []string{args.Name}, + }, nil } -func getCodehostType(repoArgs *CreateFromRepo, repoLink string) (string, error) { +func getCodehostType(repoArgs *CreateFromRepo, repoLink string) (string, *systemconfig.CodeHost, error) { if repoLink != "" { - return setting.SourceFromPublicRepo, nil + return setting.SourceFromPublicRepo, nil, nil } - ch, err := codehost.GetCodeHostInfoByID(repoArgs.CodehostID) + ch, err := systemconfig.New().GetCodeHost(repoArgs.CodehostID) if err != nil { log.Errorf("Failed to get codeHost by id %d, err: %s", repoArgs.CodehostID, err.Error()) - return "", err + return "", ch, err } - return ch.Type, nil + return ch.Type, ch, nil } -func CreateOrUpdateHelmServiceFromGitRepo(projectName string, args *HelmServiceCreationArgs, log *zap.SugaredLogger) error { +func CreateOrUpdateHelmServiceFromRepo(projectName string, args *HelmServiceCreationArgs, force bool, log *zap.SugaredLogger) (*BulkHelmServiceCreationResponse, error) { + var ( + filePaths []string + response = &BulkHelmServiceCreationResponse{} + base string + ) + resByte, resByteErr := json.Marshal(args.CreateFrom) + if resByteErr != nil { + log.Errorf("failed to json.Marshal err:%s", resByteErr) + return nil, resByteErr + } + var createFromRepo CreateFromRepo + jsonResErr := json.Unmarshal(resByte, &createFromRepo) + if jsonResErr != nil { + log.Errorf("failed to json.Unmarshal err:%s", resByteErr) + return nil, jsonResErr + } + + filePaths = createFromRepo.Paths + base = path.Join(config.S3StoragePath(), createFromRepo.Repo) + helmRenderCharts := make([]*templatemodels.RenderChart, 0, len(filePaths)) + var wg wait.Group + var mux sync.RWMutex + var serviceList []*commonmodels.Service + for _, p := range filePaths { + filePath := strings.TrimLeft(p, "/") + wg.Start(func() { + var ( + serviceName string + chartVersion string + valuesYAML []byte + finalErr error + ) + defer func() { + mux.Lock() + if finalErr != nil { + response.FailedServices = append(response.FailedServices, &FailedService{ + Path: filePath, + Error: finalErr.Error(), + }) + } else { + response.SuccessServices = append(response.SuccessServices, serviceName) + } + mux.Unlock() + }() + + currentFilePath := path.Join(base, filePath) + log.Infof("Loading chart under path %s", currentFilePath) + serviceName, chartVersion, finalErr = readChartYAMLFromLocal(currentFilePath, log) + if finalErr != nil { + return + } + valuesYAML, finalErr = readValuesYAMLFromLocal(currentFilePath, log) + if finalErr != nil { + return + } + + log.Info("Found valid chart, Starting to save and upload files") + rev, err := getNextServiceRevision(projectName, serviceName) + if err != nil { + log.Errorf("Failed to get next revision for service %s, err: %s", serviceName, err) + finalErr = e.ErrCreateTemplate.AddErr(err) + return + } + + // clear files from s3 when error occurred in next stages + defer func() { + if finalErr != nil { + clearChartFiles(projectName, serviceName, rev, log, string(args.Source)) + } + }() + + // copy to latest dir and upload to s3 + if err = commonservice.CopyAndUploadService(projectName, serviceName, currentFilePath, []string{fmt.Sprintf("%s-%d", serviceName, rev)}); err != nil { + log.Errorf("Failed to save or upload files for service %s in project %s, error: %s", serviceName, projectName, err) + finalErr = e.ErrCreateTemplate.AddErr(err) + return + } + + err = copyChartRevision(projectName, serviceName, rev) + if err != nil { + log.Errorf("Failed to copy file %s, err: %s", serviceName, err) + finalErr = errors.Wrapf(err, "Failed to copy chart info, service %s", serviceName) + return + } + + var repoLink string + if string(args.Source) == setting.SourceFromGitee { + codehostInfo, err := systemconfig.New().GetCodeHost(createFromRepo.CodehostID) + if err != nil { + finalErr = errors.Wrapf(err, "failed to get code host, id %d", createFromRepo.CodehostID) + return + } + repoLink = fmt.Sprintf("%s/%s/%s/%s/%s/%s", codehostInfo.Address, createFromRepo.Owner, createFromRepo.Repo, "tree", createFromRepo.Branch, filePath) + } + + helmServiceCreationArgs := &helmServiceCreationArgs{ + ChartName: serviceName, + ChartVersion: chartVersion, + ServiceRevision: rev, + MergedValues: string(valuesYAML), + ServiceName: serviceName, + FilePath: filePath, + ProductName: projectName, + CreateBy: args.CreatedBy, + RequestID: args.RequestID, + CodehostID: createFromRepo.CodehostID, + Owner: createFromRepo.Owner, + Namespace: createFromRepo.Namespace, + Repo: createFromRepo.Repo, + Branch: createFromRepo.Branch, + Source: string(args.Source), + RepoLink: repoLink, + GiteePath: currentFilePath, + GerritCodeHostID: createFromRepo.CodehostID, + GerritPath: currentFilePath, + GerritRepoName: createFromRepo.Repo, + GerritBranchName: createFromRepo.Branch, + GerritRemoteName: "origin", + } + + if string(args.Source) == setting.SourceFromGerrit { + helmServiceCreationArgs.GerritCodeHostID = createFromRepo.CodehostID + helmServiceCreationArgs.GerritPath = currentFilePath + helmServiceCreationArgs.GerritRepoName = createFromRepo.Repo + helmServiceCreationArgs.GerritBranchName = createFromRepo.Branch + helmServiceCreationArgs.GerritRemoteName = "origin" + } + + svc, err := createOrUpdateHelmService( + nil, + helmServiceCreationArgs, + force, + log, + ) + if err != nil { + log.Errorf("Failed to create service %s in project %s, error: %s", serviceName, projectName, err) + finalErr = e.ErrCreateTemplate.AddErr(err) + return + } + serviceList = append(serviceList, svc) + + helmRenderCharts = append(helmRenderCharts, &templatemodels.RenderChart{ + ServiceName: serviceName, + ChartVersion: svc.HelmChart.Version, + ValuesYaml: svc.HelmChart.ValuesYaml, + }) + }) + } + + wg.Wait() + + compareHelmVariable(helmRenderCharts, projectName, args.CreatedBy, log) + return response, service.AutoDeployHelmServiceToEnvs(args.CreatedBy, args.RequestID, projectName, serviceList, log) +} + +func CreateOrUpdateHelmServiceFromGitRepo(projectName string, args *HelmServiceCreationArgs, force bool, log *zap.SugaredLogger) (*BulkHelmServiceCreationResponse, error) { var err error var repoLink string repoArgs, ok := args.CreateFrom.(*CreateFromRepo) if !ok { publicArgs, ok := args.CreateFrom.(*CreateFromPublicRepo) if !ok { - return fmt.Errorf("invalid argument") + return nil, fmt.Errorf("invalid argument") } repoArgs, err = PublicRepoToPrivateRepoArgs(publicArgs) if err != nil { log.Errorf("Failed to parse repo args %+v, err: %s", publicArgs, err) - return err + return nil, err } repoLink = publicArgs.RepoLink } - source, err := getCodehostType(repoArgs, repoLink) + response := &BulkHelmServiceCreationResponse{} + source, codehostInfo, err := getCodehostType(repoArgs, repoLink) if err != nil { log.Errorf("Failed to get source form repo data %+v, err: %s", *repoArgs, err.Error()) - return err + return nil, err } helmRenderCharts := make([]*templatemodels.RenderChart, 0, len(repoArgs.Paths)) - var errs *multierror.Error var wg wait.Group var mux sync.RWMutex + serviceList := make([]*commonmodels.Service, 0) for _, p := range repoArgs.Paths { filePath := strings.TrimLeft(p, "/") wg.Start(func() { - var finalErr error - + var ( + serviceName string + chartVersion string + valuesYAML []byte + finalErr error + ) defer func() { + mux.Lock() if finalErr != nil { - mux.Lock() - errs = multierror.Append(errs, finalErr) - mux.Unlock() + response.FailedServices = append(response.FailedServices, &FailedService{ + Path: filePath, + Error: finalErr.Error(), + }) + } else { + response.SuccessServices = append(response.SuccessServices, serviceName) } + mux.Unlock() }() log.Infof("Loading chart under path %s", filePath) - var ( - serviceName string - chartVersion string - valuesYAML []byte - ) - fsTree, err := fsservice.DownloadFilesFromSource( - &fsservice.DownloadFromSourceArgs{CodehostID: repoArgs.CodehostID, Owner: repoArgs.Owner, Repo: repoArgs.Repo, Path: filePath, Branch: repoArgs.Branch, RepoLink: repoLink}, + &fsservice.DownloadFromSourceArgs{CodehostID: repoArgs.CodehostID, Owner: repoArgs.Owner, Namespace: repoArgs.Namespace, Repo: repoArgs.Repo, Path: filePath, Branch: repoArgs.Branch, RepoLink: repoLink}, func(chartTree afero.Fs) (string, error) { var err error serviceName, chartVersion, err = readChartYAML(afero.NewIOFS(chartTree), filepath.Base(filePath), log) if err != nil { return serviceName, err } - valuesYAML, _, err = readValuesYAML(afero.NewIOFS(chartTree), filepath.Base(filePath), log) + valuesYAML, err = readValuesYAML(afero.NewIOFS(chartTree), filepath.Base(filePath), log) return serviceName, err }) if err != nil { @@ -434,30 +925,58 @@ func CreateOrUpdateHelmServiceFromGitRepo(projectName string, args *HelmServiceC log.Info("Found valid chart, Starting to save and upload files") + rev, err := getNextServiceRevision(projectName, serviceName) + if err != nil { + log.Errorf("Failed to get next revision for service %s, err: %s", serviceName, err) + finalErr = e.ErrCreateTemplate.AddErr(err) + return + } + + // clear files from both s3 and local when error occurred in next stages + defer func() { + if finalErr != nil { + clearChartFiles(projectName, serviceName, rev, log) + } + }() + // save files to disk and upload them to s3 - if err = commonservice.SaveAndUploadService(projectName, serviceName, fsTree); err != nil { + if err = commonservice.SaveAndUploadService(projectName, serviceName, []string{fmt.Sprintf("%s-%d", serviceName, rev)}, fsTree); err != nil { log.Errorf("Failed to save or upload files for service %s in project %s, error: %s", serviceName, projectName, err) finalErr = e.ErrCreateTemplate.AddErr(err) return } + err = copyChartRevision(projectName, serviceName, rev) + if err != nil { + log.Errorf("Failed to copy file %s, err: %s", serviceName, err) + finalErr = errors.Wrapf(err, "Failed to copy chart info, service %s", serviceName) + return + } + + if source != setting.SourceFromPublicRepo && codehostInfo != nil { + repoLink = fmt.Sprintf("%s/%s/%s/%s/%s/%s", codehostInfo.Address, repoArgs.Namespace, repoArgs.Repo, "tree", repoArgs.Branch, filePath) + } + svc, err := createOrUpdateHelmService( fsTree, &helmServiceCreationArgs{ - ChartName: serviceName, - ChartVersion: chartVersion, - ValuesYAML: string(valuesYAML), - ServiceName: serviceName, - FilePath: filePath, - ProductName: projectName, - CreateBy: args.CreatedBy, - CodehostID: repoArgs.CodehostID, - Owner: repoArgs.Owner, - Repo: repoArgs.Repo, - Branch: repoArgs.Branch, - RepoLink: repoLink, - Source: source, - }, + ChartName: serviceName, + ChartVersion: chartVersion, + ServiceRevision: rev, + MergedValues: string(valuesYAML), + ServiceName: serviceName, + FilePath: filePath, + ProductName: projectName, + CreateBy: args.CreatedBy, + RequestID: args.RequestID, + CodehostID: repoArgs.CodehostID, + Owner: repoArgs.Owner, + Namespace: repoArgs.Namespace, + Repo: repoArgs.Repo, + Branch: repoArgs.Branch, + RepoLink: repoLink, + Source: source, + }, force, log, ) if err != nil { @@ -465,34 +984,32 @@ func CreateOrUpdateHelmServiceFromGitRepo(projectName string, args *HelmServiceC finalErr = e.ErrCreateTemplate.AddErr(err) return } + serviceList = append(serviceList, svc) - if svc.HelmChart != nil { - helmRenderCharts = append(helmRenderCharts, &templatemodels.RenderChart{ - ServiceName: serviceName, - ChartVersion: svc.HelmChart.Version, - ValuesYaml: svc.HelmChart.ValuesYaml, - }) - } + helmRenderCharts = append(helmRenderCharts, &templatemodels.RenderChart{ + ServiceName: serviceName, + ChartVersion: svc.HelmChart.Version, + ValuesYaml: svc.HelmChart.ValuesYaml, + }) }) } wg.Wait() compareHelmVariable(helmRenderCharts, projectName, args.CreatedBy, log) - - return errs.ErrorOrNil() + return response, service.AutoDeployHelmServiceToEnvs(args.CreatedBy, args.RequestID, projectName, serviceList, log) } -func CreateOrUpdateBulkHelmService(projectName string, args *BulkHelmServiceCreationArgs, logger *zap.SugaredLogger) (*BulkHelmServiceCreationResponse, error) { +func CreateOrUpdateBulkHelmService(projectName string, args *BulkHelmServiceCreationArgs, force bool, logger *zap.SugaredLogger) (*BulkHelmServiceCreationResponse, error) { switch args.Source { case LoadFromChartTemplate: - return CreateOrUpdateBulkHelmServiceFromTemplate(projectName, args, logger) + return CreateOrUpdateBulkHelmServiceFromTemplate(projectName, args, force, logger) default: return nil, fmt.Errorf("invalid source") } } -func CreateOrUpdateBulkHelmServiceFromTemplate(projectName string, args *BulkHelmServiceCreationArgs, logger *zap.SugaredLogger) (*BulkHelmServiceCreationResponse, error) { +func CreateOrUpdateBulkHelmServiceFromTemplate(projectName string, args *BulkHelmServiceCreationArgs, force bool, logger *zap.SugaredLogger) (*BulkHelmServiceCreationResponse, error) { templateArgs, ok := args.CreateFrom.(*CreateFromChartTemplate) if !ok { return nil, fmt.Errorf("invalid argument") @@ -515,6 +1032,8 @@ func CreateOrUpdateBulkHelmServiceFromTemplate(projectName string, args *BulkHel //record errors for every service failedServiceMap := &sync.Map{} renderChartMap := &sync.Map{} + svcMap := &sync.Map{} + serviceList := make([]*commonmodels.Service, 0) wg := sync.WaitGroup{} // run goroutines to speed up @@ -522,10 +1041,11 @@ func CreateOrUpdateBulkHelmServiceFromTemplate(projectName string, args *BulkHel wg.Add(1) go func(repoConfig *commonservice.RepoConfig, path string) { defer wg.Done() - renderChart, err := handleSingleService(projectName, repoConfig, path, from, args.CreatedBy, templateChartData, logger) + renderChart, svcInfo, err := handleSingleService(projectName, repoConfig, path, from, args, templateChartData, force, logger) if err != nil { failedServiceMap.Store(path, err.Error()) } else { + svcMap.Store(renderChart.ServiceName, svcInfo) renderChartMap.Store(renderChart.ServiceName, renderChart) } }(args.ValuesData.GitRepoConfig, singlePath) @@ -545,6 +1065,10 @@ func CreateOrUpdateBulkHelmServiceFromTemplate(projectName string, args *BulkHel renderChars = append(renderChars, value.(*templatemodels.RenderChart)) return true }) + svcMap.Range(func(key, value interface{}) bool { + serviceList = append(serviceList, value.(*commonmodels.Service)) + return true + }) failedServiceMap.Range(func(key, value interface{}) bool { resp.FailedServices = append(resp.FailedServices, &FailedService{ @@ -555,85 +1079,115 @@ func CreateOrUpdateBulkHelmServiceFromTemplate(projectName string, args *BulkHel }) compareHelmVariable(renderChars, projectName, args.CreatedBy, logger) - - return resp, nil + return resp, service.AutoDeployHelmServiceToEnvs(args.CreatedBy, args.RequestID, projectName, serviceList, logger) } -func handleSingleService(projectName string, repoConfig *commonservice.RepoConfig, path, fromPath, createBy string, - templateChartData *ChartTemplateData, logger *zap.SugaredLogger) (*templatemodels.RenderChart, error) { - +func handleSingleService(projectName string, repoConfig *commonservice.RepoConfig, path, fromPath string, args *BulkHelmServiceCreationArgs, + templateChartData *ChartTemplateData, force bool, logger *zap.SugaredLogger) (*templatemodels.RenderChart, *commonmodels.Service, error) { valuesYAML, err := fsservice.DownloadFileFromSource(&fsservice.DownloadFromSourceArgs{ CodehostID: repoConfig.CodehostID, Owner: repoConfig.Owner, Repo: repoConfig.Repo, + Namespace: repoConfig.Namespace, Path: path, Branch: repoConfig.Branch, }) if err != nil { - return nil, err + return nil, nil, err } if len(valuesYAML) == 0 { - return nil, fmt.Errorf("values.yaml is empty") + return nil, nil, fmt.Errorf("values.yaml is empty") } values := [][]byte{templateChartData.DefaultValuesYAML, valuesYAML} mergedValues, err := yamlutil.Merge(values) if err != nil { logger.Errorf("Failed to merge values, err: %s", err) - return nil, err + return nil, nil, err } serviceName := filepath.Base(path) serviceName = strings.TrimSuffix(serviceName, filepath.Ext(serviceName)) + serviceName = strings.TrimSpace(serviceName) to := filepath.Join(config.LocalServicePath(projectName, serviceName), serviceName) + // remove old files + if err = os.RemoveAll(to); err != nil { + logger.Errorf("Failed to remove dir %s, err: %s", to, err) + return nil, nil, err + } if err = copy.Copy(fromPath, to); err != nil { logger.Errorf("Failed to copy file from %s to %s, err: %s", fromPath, to, err) - return nil, err + return nil, nil, err } // write values.yaml file if err = os.WriteFile(filepath.Join(to, setting.ValuesYaml), mergedValues, 0644); err != nil { logger.Errorf("Failed to write values, err: %s", err) - return nil, err + return nil, nil, err + } + + rev, err := getNextServiceRevision(projectName, serviceName) + if err != nil { + log.Errorf("Failed to get next revision for service %s, err: %s", serviceName, err) + return nil, nil, errors.Wrapf(err, "Failed to get service next revision, service %s", serviceName) + } + + err = copyChartRevision(projectName, serviceName, rev) + if err != nil { + log.Errorf("Failed to copy file %s, err: %s", serviceName, err) + return nil, nil, errors.Wrapf(err, "Failed to copy chart info, service %s", serviceName) } fsTree := os.DirFS(config.LocalServicePath(projectName, serviceName)) - ServiceS3Base := config.ObjectStorageServicePath(projectName, serviceName) - if err = fsservice.ArchiveAndUploadFilesToS3(fsTree, serviceName, ServiceS3Base, logger); err != nil { + serviceS3Base := config.ObjectStorageServicePath(projectName, serviceName) + + // clear files from both s3 and local when error occurred in next stages + defer func() { + if err != nil { + clearChartFiles(projectName, serviceName, rev, logger) + } + }() + + if err = fsservice.ArchiveAndUploadFilesToS3(fsTree, []string{serviceName, fmt.Sprintf("%s-%d", serviceName, rev)}, serviceS3Base, logger); err != nil { logger.Errorf("Failed to upload files for service %s in project %s, err: %s", serviceName, projectName, err) - return nil, err + return nil, nil, err } - _, err = createOrUpdateHelmService( + svc, err := createOrUpdateHelmService( fsTree, &helmServiceCreationArgs{ ChartName: templateChartData.ChartName, ChartVersion: templateChartData.ChartVersion, - ValuesYAML: string(mergedValues), + ServiceRevision: rev, + MergedValues: string(mergedValues), ServiceName: serviceName, FilePath: to, ProductName: projectName, - CreateBy: createBy, + CreateBy: args.CreatedBy, + RequestID: args.RequestID, CodehostID: repoConfig.CodehostID, Source: setting.SourceFromChartTemplate, HelmTemplateName: templateChartData.TemplateName, ValuePaths: []string{path}, - ValuesYaml: string(templateChartData.DefaultValuesYAML), + ValuesYaml: string(valuesYAML), + AutoSync: args.AutoSync, + ValuesSource: args.ValuesData, }, + force, logger, ) if err != nil { logger.Errorf("Failed to create service %s in project %s, error: %s", serviceName, projectName, err) - return nil, err + return nil, nil, err } return &templatemodels.RenderChart{ ServiceName: serviceName, ChartVersion: templateChartData.ChartVersion, ValuesYaml: string(mergedValues), - }, nil + }, svc, nil } func readChartYAML(chartTree fs.FS, base string, logger *zap.SugaredLogger) (string, string, error) { @@ -651,20 +1205,37 @@ func readChartYAML(chartTree fs.FS, base string, logger *zap.SugaredLogger) (str return chart.Name, chart.Version, nil } -func readValuesYAML(chartTree fs.FS, base string, logger *zap.SugaredLogger) ([]byte, map[string]interface{}, error) { +func readChartYAMLFromLocal(base string, logger *zap.SugaredLogger) (string, string, error) { + chartFile, err := util.ReadFile(filepath.Join(base, setting.ChartYaml)) + if err != nil { + logger.Errorf("Failed to read %s, err: %s", setting.ChartYaml, err) + return "", "", err + } + chart := new(Chart) + if err = yaml.Unmarshal(chartFile, chart); err != nil { + log.Errorf("Failed to unmarshal yaml %s, err: %s", setting.ChartYaml, err) + return "", "", err + } + + return chart.Name, chart.Version, nil +} + +func readValuesYAML(chartTree fs.FS, base string, logger *zap.SugaredLogger) ([]byte, error) { content, err := fs.ReadFile(chartTree, filepath.Join(base, setting.ValuesYaml)) if err != nil { logger.Errorf("Failed to read %s, err: %s", setting.ValuesYaml, err) - return nil, nil, err + return nil, err } + return content, nil +} - valuesMap := make(map[string]interface{}) - if err = yaml.Unmarshal(content, &valuesMap); err != nil { - logger.Errorf("Failed to unmarshal yaml %s, err: %s", setting.ValuesYaml, err) - return nil, nil, err +func readValuesYAMLFromLocal(base string, logger *zap.SugaredLogger) ([]byte, error) { + content, err := util.ReadFile(filepath.Join(base, setting.ValuesYaml)) + if err != nil { + logger.Errorf("Failed to read %s, err: %s", setting.ValuesYaml, err) + return nil, err } - - return content, valuesMap, nil + return content, nil } func geneCreationDetail(args *helmServiceCreationArgs) interface{} { @@ -672,14 +1243,15 @@ func geneCreationDetail(args *helmServiceCreationArgs) interface{} { case setting.SourceFromGitlab, setting.SourceFromGithub, setting.SourceFromGerrit, - setting.SourceFromCodeHub, - setting.SourceFromIlyshin: + setting.SourceFromGitee, + setting.SourceFromCodeHub: return &models.CreateFromRepo{ GitRepoConfig: &templatemodels.GitRepoConfig{ CodehostID: args.CodehostID, Owner: args.Owner, Repo: args.Repo, Branch: args.Branch, + Namespace: args.Namespace, }, LoadPath: args.FilePath, } @@ -689,95 +1261,192 @@ func geneCreationDetail(args *helmServiceCreationArgs) interface{} { LoadPath: args.FilePath, } case setting.SourceFromChartTemplate: - var yamlData *template.CustomYaml - if args.CodehostID > 0 { - yamlData = &template.CustomYaml{ - YamlSource: setting.ValuesYamlSourceGitRepo, - YamlContent: "", + yamlData := &templatemodels.CustomYaml{ + YamlContent: args.ValuesYaml, + } + variables := make([]*models.Variable, 0, len(args.Variables)) + for _, variable := range args.Variables { + variables = append(variables, &models.Variable{ + Key: variable.Key, + Value: variable.Value, + }) + } + if args.ValuesSource != nil && args.ValuesSource.GitRepoConfig != nil { + //yamlData.Source = args.ValuesSource.YamlSource + yamlData.Source = setting.SourceFromGitRepo + repoData := &models.CreateFromRepo{ GitRepoConfig: &templatemodels.GitRepoConfig{ - CodehostID: args.CodehostID, - Owner: args.Owner, - Repo: args.Repo, - Branch: args.Branch, + CodehostID: args.ValuesSource.GitRepoConfig.CodehostID, + Owner: args.ValuesSource.GitRepoConfig.Owner, + Repo: args.ValuesSource.GitRepoConfig.Repo, + Branch: args.ValuesSource.GitRepoConfig.Branch, + Namespace: args.ValuesSource.GitRepoConfig.Namespace, }, - ValuesPaths: args.ValuePaths, } - } else if len(args.ValuesYaml) > 0 { - yamlData = &template.CustomYaml{ - YamlSource: setting.ValuesYamlSourceFreeEdit, - YamlContent: args.ValuesYaml, + if len(args.ValuesSource.GitRepoConfig.ValuesPaths) > 0 { + repoData.LoadPath = args.ValuesSource.GitRepoConfig.ValuesPaths[0] } + yamlData.SourceDetail = repoData } - return &models.CreateFromChartTemplate{ YamlData: yamlData, TemplateName: args.HelmTemplateName, ServiceName: args.ServiceName, + Variables: variables, + } + case setting.SourceFromChartRepo: + return models.CreateFromChartRepo{ + ChartRepoName: args.ChartRepoName, + ChartName: args.ChartName, + ChartVersion: args.ChartVersion, } } return nil } -func createOrUpdateHelmService(fsTree fs.FS, args *helmServiceCreationArgs, logger *zap.SugaredLogger) (*models.Service, error) { - chartName, chartVersion, err := readChartYAML(fsTree, args.ServiceName, logger) +func renderVariablesToYaml(valuesYaml string, productName, serviceName string, variables []*Variable) (string, error) { + valuesYaml = strings.Replace(valuesYaml, setting.TemplateVariableProduct, productName, -1) + valuesYaml = strings.Replace(valuesYaml, setting.TemplateVariableService, serviceName, -1) + + // build replace data + valuesMap := make(map[string]interface{}) + for _, variable := range variables { + valuesMap[variable.Key] = variable.Value + } + + tmpl, err := template.New("values").Parse(valuesYaml) if err != nil { - logger.Errorf("Failed to read chart.yaml, err %s", err) - return nil, err + log.Errorf("failed to parse template, err %s valuesYaml %s", err, valuesYaml) + return "", errors.Wrapf(err, "failed to parse template, err %s", err) } - values, valuesMap, err := readValuesYAML(fsTree, args.ServiceName, logger) + buf := bytes.NewBufferString("") + err = tmpl.Execute(buf, valuesMap) if err != nil { - logger.Errorf("Failed to read values.yaml, err %s", err) - return nil, err + log.Errorf("failed to render values content, err %s", err) + return "", fmt.Errorf("failed to render variables") + } + valuesYaml = buf.String() + return valuesYaml, nil +} + +func createOrUpdateHelmService(fsTree fs.FS, args *helmServiceCreationArgs, force bool, logger *zap.SugaredLogger) (*commonmodels.Service, error) { + var ( + chartName, chartVersion string + err error + ) + switch args.Source { + case string(LoadFromGerrit): + base := path.Join(config.S3StoragePath(), args.GerritRepoName) + chartName, chartVersion, err = readChartYAMLFromLocal(filepath.Join(base, args.FilePath), logger) + case setting.SourceFromGitee: + base := path.Join(config.S3StoragePath(), args.Repo) + chartName, chartVersion, err = readChartYAMLFromLocal(filepath.Join(base, args.FilePath), logger) + default: + chartName, chartVersion, err = readChartYAML(fsTree, args.ServiceName, logger) } - valuesYaml := string(values) - serviceTemplate := fmt.Sprintf(setting.ServiceTemplateCounterName, args.ServiceName, args.ProductName) - rev, err := commonrepo.NewCounterColl().GetNextSeq(serviceTemplate) if err != nil { - logger.Errorf("Failed to get next revision for service %s, err: %s", args.ServiceName, err) + logger.Errorf("Failed to read chart.yaml, err %s", err) return nil, err } - if err = commonrepo.NewServiceColl().Delete(args.ServiceName, setting.HelmDeployType, args.ProductName, setting.ProductStatusDeleting, rev); err != nil { - logger.Warnf("Failed to delete stale service %s with revision %d, err: %s", args.ServiceName, rev, err) - } - containerList, err := commonservice.ParseImagesForProductService(valuesMap, args.ServiceName, args.ProductName) + valuesYaml := args.MergedValues + valuesMap := make(map[string]interface{}) + err = yaml.Unmarshal([]byte(valuesYaml), &valuesMap) if err != nil { - return nil, errors.Wrapf(err, "failed to parse service from yaml") + logger.Errorf("Failed to unmarshall yaml, err %s", err) + return nil, err } - serviceObj := &models.Service{ - ServiceName: args.ServiceName, - Type: setting.HelmDeployType, - Revision: rev, - ProductName: args.ProductName, - Visibility: setting.PrivateVisibility, - CreateTime: time.Now().Unix(), - CreateBy: args.CreateBy, - Containers: containerList, - CodehostID: args.CodehostID, - RepoOwner: args.Owner, - RepoName: args.Repo, - BranchName: args.Branch, - LoadPath: args.FilePath, - SrcPath: args.RepoLink, - CreateFrom: geneCreationDetail(args), - Source: args.Source, - HelmChart: &models.HelmChart{ + containerList, err := commonservice.ParseImagesForProductService(valuesMap, args.ServiceName, args.ProductName) + if err != nil { + return nil, errors.Wrapf(err, "Failed to parse service from yaml") + } + + serviceObj := &commonmodels.Service{ + ServiceName: args.ServiceName, + Type: setting.HelmDeployType, + Revision: args.ServiceRevision, + ProductName: args.ProductName, + Visibility: setting.PrivateVisibility, + CreateTime: time.Now().Unix(), + CreateBy: args.CreateBy, + Containers: containerList, + CodehostID: args.CodehostID, + RepoOwner: args.Owner, + RepoNamespace: args.Namespace, + RepoName: args.Repo, + BranchName: args.Branch, + LoadPath: args.FilePath, + SrcPath: args.RepoLink, + Source: args.Source, + ReleaseNaming: setting.DefaultReleaseNaming, + AutoSync: args.AutoSync, + HelmChart: &commonmodels.HelmChart{ Name: chartName, Version: chartVersion, ValuesYaml: valuesYaml, }, } + if args.CreationDetail != nil { + serviceObj.CreateFrom = args.CreationDetail + } else { + serviceObj.CreateFrom = geneCreationDetail(args) + } - log.Infof("Starting to create service %s with revision %d", args.ServiceName, rev) + switch args.Source { + case string(LoadFromGerrit): + serviceObj.GerritPath = args.GerritPath + serviceObj.GerritCodeHostID = args.GerritCodeHostID + serviceObj.GerritRepoName = args.GerritRepoName + serviceObj.GerritBranchName = args.GerritBranchName + serviceObj.GerritRemoteName = args.GerritRemoteName + case setting.SourceFromGitee: + serviceObj.GiteePath = args.GiteePath + } + + log.Infof("Starting to create service %s with revision %d", args.ServiceName, args.ServiceRevision) + currentSvcTmpl, err := commonrepo.NewServiceColl().Find(&commonrepo.ServiceFindOption{ + ProductName: args.ProductName, + ServiceName: args.ServiceName, + ExcludeStatus: setting.ProductStatusDeleting, + IgnoreNoDocumentErr: true, + }) + if err != nil { + log.Errorf("Failed to find current service template %s error: %s", args.ServiceName, err) + return nil, err + } + + // update status of current service template to deleting + if currentSvcTmpl != nil { + if !force { + return nil, fmt.Errorf("service:%s already exists", args.ServiceName) + } + err = commonrepo.NewServiceColl().UpdateStatus(args.ServiceName, args.ProductName, setting.ProductStatusDeleting) + if err != nil { + log.Errorf("Failed to set status of current service templates, serviceName: %s, err: %s", args.ServiceName, err) + return nil, err + } + serviceObj.ReleaseNaming = currentSvcTmpl.GetReleaseNaming() + } + // create new service template if err = commonrepo.NewServiceColl().Create(serviceObj); err != nil { log.Errorf("Failed to create service %s error: %s", args.ServiceName, err) return nil, err } + switch args.Source { + case string(LoadFromGerrit): + if err := createGerritWebhookByService(args.CodehostID, args.ServiceName, args.Repo, args.Branch); err != nil { + log.Errorf("Failed to create gerrit webhook, err: %s", err) + return nil, err + } + default: + commonservice.ProcessServiceWebhook(serviceObj, currentSvcTmpl, args.ServiceName, logger) + } + if err = templaterepo.NewProductColl().AddService(args.ProductName, args.ServiceName); err != nil { log.Errorf("Failed to add service %s to project %s, err: %s", args.ProductName, args.ServiceName, err) return nil, err @@ -786,9 +1455,7 @@ func createOrUpdateHelmService(fsTree fs.FS, args *helmServiceCreationArgs, logg return serviceObj, nil } -func loadServiceFileInfos(productName, serviceName, dir string) ([]*types.FileInfo, error) { - base := config.LocalServicePath(productName, serviceName) - +func loadServiceFileInfos(productName, serviceName string, revision int64, dir string) ([]*types.FileInfo, error) { svc, err := commonrepo.NewServiceColl().Find(&commonrepo.ServiceFindOption{ ProductName: productName, ServiceName: serviceName, @@ -797,11 +1464,29 @@ func loadServiceFileInfos(productName, serviceName, dir string) ([]*types.FileIn return nil, e.ErrFilePath.AddDesc(err.Error()) } + base := config.LocalServicePath(productName, serviceName) + if revision > 0 { + base = config.LocalServicePathWithRevision(productName, serviceName, revision) + if err = commonservice.PreloadServiceManifestsByRevision(base, svc); err != nil { + log.Warnf("failed to get chart of revision: %d for service: %s, use latest version", + svc.Revision, svc.ServiceName) + } + } + if err != nil || revision == 0 { + base = config.LocalServicePath(productName, serviceName) + err = commonservice.PreLoadServiceManifests(base, svc) + if err != nil { + return nil, e.ErrFilePath.AddDesc(err.Error()) + } + } + err = commonservice.PreLoadServiceManifests(base, svc) if err != nil { return nil, e.ErrFilePath.AddDesc(err.Error()) } + var fis []*types.FileInfo + files, err := os.ReadDir(filepath.Join(base, serviceName, dir)) if err != nil { return nil, e.ErrFilePath.AddDesc(err.Error()) @@ -826,11 +1511,10 @@ func loadServiceFileInfos(productName, serviceName, dir string) ([]*types.FileIn return fis, nil } -// UpdateHelmService TODO need to be deprecated -func UpdateHelmService(args *HelmServiceArgs, log *zap.SugaredLogger) error { - var serviceNames []string +// EditHelmService deprecated +func EditHelmService(args *HelmServiceArgs, log *zap.SugaredLogger) error { + serviceMap := make(map[string]int64) for _, helmServiceInfo := range args.HelmServiceInfos { - serviceNames = append(serviceNames, helmServiceInfo.ServiceName) opt := &commonrepo.ServiceFindOption{ ProductName: args.ProductName, @@ -854,6 +1538,7 @@ func UpdateHelmService(args *HelmServiceArgs, log *zap.SugaredLogger) error { } // TODO：use yaml compare instead of just comparing the characters + // TODO service variables if helmServiceInfo.FileName == setting.ValuesYaml && preServiceTmpl.HelmChart.ValuesYaml != helmServiceInfo.FileContent { var valuesMap map[string]interface{} if err = yaml.Unmarshal([]byte(helmServiceInfo.FileContent), &valuesMap); err != nil { @@ -912,6 +1597,7 @@ func UpdateHelmService(args *HelmServiceArgs, log *zap.SugaredLogger) error { return fmt.Errorf("get next helm service revision error: %v", err) } + serviceMap[helmServiceInfo.ServiceName] = rev preServiceTmpl.Revision = rev if err := commonrepo.NewServiceColl().Delete(helmServiceInfo.ServiceName, setting.HelmDeployType, args.ProductName, setting.ProductStatusDeleting, preServiceTmpl.Revision); err != nil { log.Errorf("helmService.update delete %s error: %v", helmServiceInfo.ServiceName, err) @@ -923,9 +1609,9 @@ func UpdateHelmService(args *HelmServiceArgs, log *zap.SugaredLogger) error { } } - for _, serviceName := range serviceNames { + for serviceName, rev := range serviceMap { s3Base := config.ObjectStorageServicePath(args.ProductName, serviceName) - if err := fsservice.ArchiveAndUploadFilesToS3(os.DirFS(config.LocalServicePath(args.ProductName, serviceName)), serviceName, s3Base, log); err != nil { + if err := fsservice.ArchiveAndUploadFilesToS3(os.DirFS(config.LocalServicePath(args.ProductName, serviceName)), []string{serviceName, fmt.Sprintf("%s-%d", serviceName, rev)}, s3Base, log); err != nil { return e.ErrUpdateTemplate.AddDesc(err.Error()) } } @@ -937,7 +1623,7 @@ func UpdateHelmService(args *HelmServiceArgs, log *zap.SugaredLogger) error { func compareHelmVariable(chartInfos []*templatemodels.RenderChart, productName, createdBy string, log *zap.SugaredLogger) { // 对比上个版本的renderset，新增一个版本 latestChartInfos := make([]*templatemodels.RenderChart, 0) - renderOpt := &commonrepo.RenderSetFindOption{Name: productName} + renderOpt := &commonrepo.RenderSetFindOption{Name: productName, ProductTmpl: productName, IsDefault: true} if latestDefaultRenderSet, err := commonrepo.NewRenderSetColl().Find(renderOpt); err == nil { latestChartInfos = latestDefaultRenderSet.ChartInfos } diff --git a/pkg/microservice/aslan/core/service/service/loader.go b/pkg/microservice/aslan/core/service/service/loader.go index 6fc46294e92186b35e846e6eff6c265bc4650d00..9dffa729e4bffd40c07604be89f73cc34bc49fec 100644 --- a/pkg/microservice/aslan/core/service/service/loader.go +++ b/pkg/microservice/aslan/core/service/service/loader.go @@ -23,7 +23,6 @@ import ( "io/ioutil" "os" "path" - "path/filepath" "strings" "github.com/google/go-github/v35/github" @@ -35,12 +34,11 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/command" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/codehost" - "github.com/koderover/zadig/pkg/shared/poetry" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" "github.com/koderover/zadig/pkg/tool/codehub" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/gerrit" - "github.com/koderover/zadig/pkg/tool/ilyshin" + "github.com/koderover/zadig/pkg/tool/gitee" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/util" ) @@ -55,7 +53,7 @@ type LoadServiceReq struct { func PreloadServiceFromCodeHost(codehostID int, repoOwner, repoName, repoUUID, branchName, remoteName, path string, isDir bool, log *zap.SugaredLogger) ([]string, error) { var ret []string - ch, err := codehost.GetCodeHostInfoByID(codehostID) + ch, err := systemconfig.New().GetCodeHost(codehostID) if err != nil { log.Errorf("Failed to load codehost for preload service list, the error is: %+v", err) return nil, e.ErrPreloadServiceTemplate.AddDesc(err.Error()) @@ -67,8 +65,8 @@ func PreloadServiceFromCodeHost(codehostID int, repoOwner, repoName, repoUUID, b ret, err = preloadGerritService(ch, repoName, branchName, remoteName, path, isDir) case setting.SourceFromCodeHub: ret, err = preloadCodehubService(ch, repoName, repoUUID, branchName, path, isDir) - case setting.SourceFromIlyshin: - ret, err = preloadIlyshinService(ch, repoOwner, repoName, branchName, path, isDir) + case setting.SourceFromGitee: + ret, err = preloadGiteeService(ch, repoOwner, repoName, branchName, remoteName, path, isDir) default: return nil, e.ErrPreloadServiceTemplate.AddDesc("Not supported code source") } @@ -77,21 +75,21 @@ func PreloadServiceFromCodeHost(codehostID int, repoOwner, repoName, repoUUID, b } // LoadServiceFromCodeHost 根据提供的codehost信息加载服务 -func LoadServiceFromCodeHost(username string, codehostID int, repoOwner, repoName, repoUUID, branchName, remoteName string, args *LoadServiceReq, log *zap.SugaredLogger) error { - ch, err := codehost.GetCodeHostInfoByID(codehostID) +func LoadServiceFromCodeHost(username string, codehostID int, repoOwner, namespace, repoName, repoUUID, branchName, remoteName string, args *LoadServiceReq, force bool, log *zap.SugaredLogger) error { + ch, err := systemconfig.New().GetCodeHost(codehostID) if err != nil { log.Errorf("Failed to load codehost for preload service list, the error is: %+v", err) return e.ErrLoadServiceTemplate.AddDesc(err.Error()) } switch ch.Type { case setting.SourceFromGithub, setting.SourceFromGitlab: - return loadService(username, ch, repoOwner, repoName, branchName, args, log) + return loadService(username, ch, repoOwner, namespace, repoName, branchName, args, force, log) case setting.SourceFromGerrit: - return loadGerritService(username, ch, repoOwner, repoName, branchName, remoteName, args, log) + return loadGerritService(username, ch, repoOwner, repoName, branchName, remoteName, args, force, log) case setting.SourceFromCodeHub: - return loadCodehubService(username, ch, repoOwner, repoName, repoUUID, branchName, args, log) - case setting.SourceFromIlyshin: - return loadIlyshinService(username, ch, repoOwner, repoName, branchName, args, log) + return loadCodehubService(username, ch, repoOwner, repoName, repoUUID, branchName, args, force, log) + case setting.SourceFromGitee: + return loadGiteeService(username, ch, repoOwner, repoName, branchName, remoteName, args, force, log) default: return e.ErrLoadServiceTemplate.AddDesc("unsupported code source") } @@ -99,7 +97,7 @@ func LoadServiceFromCodeHost(username string, codehostID int, repoOwner, repoNam // ValidateServiceUpdate 根据服务名和提供的加载信息确认是否可以更新服务加载地址 func ValidateServiceUpdate(codehostID int, serviceName, repoOwner, repoName, repoUUID, branchName, remoteName, path string, isDir bool, log *zap.SugaredLogger) error { - detail, err := codehost.GetCodeHostInfoByID(codehostID) + detail, err := systemconfig.New().GetCodeHost(codehostID) if err != nil { log.Errorf("Failed to load codehost for validate service update, the error is: %+v", err) return e.ErrValidateServiceUpdate.AddDesc(err.Error()) @@ -113,28 +111,20 @@ func ValidateServiceUpdate(codehostID int, serviceName, repoOwner, repoName, rep return validateServiceUpdateGerrit(detail, serviceName, repoName, branchName, remoteName, path, isDir) case setting.SourceFromCodeHub: return validateServiceUpdateCodehub(detail, serviceName, repoName, repoUUID, branchName, path, isDir) - case setting.SourceFromIlyshin: - return validateServiceUpdateIlyshin(detail, serviceName, repoOwner, repoName, branchName, path, isDir) + case setting.SourceFromGitee: + return validateServiceUpdateGitee(detail, serviceName, repoOwner, repoName, branchName, remoteName, path, isDir) default: return e.ErrValidateServiceUpdate.AddDesc("Not supported code source") } } // 根据repo信息获取gerrit可以加载的服务列表 -func preloadGerritService(detail *poetry.CodeHost, repoName, branchName, remoteName, loadPath string, isDir bool) ([]string, error) { +func preloadGerritService(detail *systemconfig.CodeHost, repoName, branchName, remoteName, loadPath string, isDir bool) ([]string, error) { ret := make([]string, 0) base := path.Join(config.S3StoragePath(), repoName) if _, err := os.Stat(base); os.IsNotExist(err) { - chDetail := &codehost.Detail{ - ID: detail.ID, - Name: "", - Address: detail.Address, - Owner: detail.Namespace, - Source: detail.Type, - OauthToken: detail.AccessToken, - } - err = command.RunGitCmds(chDetail, setting.GerritDefaultOwner, repoName, branchName, remoteName) + err = command.RunGitCmds(detail, setting.GerritDefaultOwner, setting.GerritDefaultOwner, repoName, branchName, remoteName) if err != nil { return nil, e.ErrPreloadServiceTemplate.AddDesc(err.Error()) } @@ -144,7 +134,7 @@ func preloadGerritService(detail *poetry.CodeHost, repoName, branchName, remoteN if !isDir { if !isYaml(loadPath) { - log.Errorf("trying to preload a non-yaml file") + log.Error("trying to preload a non-yaml file") return nil, e.ErrPreloadServiceTemplate.AddDesc("Non-yaml service loading is not supported") } pathSegment := strings.Split(loadPath, "/") @@ -153,10 +143,10 @@ func preloadGerritService(detail *poetry.CodeHost, repoName, branchName, remoteN } else { fileInfos, err := ioutil.ReadDir(filePath) if err != nil { - log.Errorf("Failed to read directory info of path: %s, the error is: %+v", filePath, err) + log.Error("Failed to read directory info of path: %s, the error is: %+v", filePath, err) return nil, e.ErrPreloadServiceTemplate.AddDesc(err.Error()) } - if isValidGerritServiceDir(fileInfos) { + if isValidServiceDir(fileInfos) { svcName := loadPath if loadPath == "" { svcName = repoName @@ -175,7 +165,7 @@ func preloadGerritService(detail *poetry.CodeHost, repoName, branchName, remoteN log.Errorf("Failed to get subdir content from gerrit with path: %s, the error is: %+v", subDirPath, err) return nil, e.ErrPreloadServiceTemplate.AddDesc(err.Error()) } - if isValidGerritServiceDir(subtree) { + if isValidServiceDir(subtree) { ret = append(ret, getFileName(file.Name())) isGrandParent = true } @@ -190,10 +180,10 @@ func preloadGerritService(detail *poetry.CodeHost, repoName, branchName, remoteN } // 根据 repo 信息获取 codehub 可以加载的服务列表 -func preloadCodehubService(detail *poetry.CodeHost, repoName, repoUUID, branchName, path string, isDir bool) ([]string, error) { +func preloadCodehubService(detail *systemconfig.CodeHost, repoName, repoUUID, branchName, path string, isDir bool) ([]string, error) { var ret []string - codeHubClient := codehub.NewCodeHubClient(detail.AccessKey, detail.SecretKey, detail.Region) + codeHubClient := codehub.NewCodeHubClient(detail.AccessKey, detail.SecretKey, detail.Region, config.ProxyHTTPSAddr(), detail.EnableProxy) // 非文件夹情况下直接获取文件信息 if !isDir { if !isYaml(path) { @@ -246,75 +236,82 @@ func preloadCodehubService(detail *poetry.CodeHost, repoName, repoUUID, branchNa return ret, nil } -// 根据 repo 信息获取 ilyshin 可以加载的服务列表 -func preloadIlyshinService(detail *poetry.CodeHost, repoOwner, repoName, branchName, path string, isDir bool) ([]string, error) { +//Get a list of services that gitee can load based on repo information +func preloadGiteeService(detail *systemconfig.CodeHost, repoOwner, repoName, branchName, remoteName, loadPath string, isDir bool) ([]string, error) { ret := make([]string, 0) - ilyshinClient := ilyshin.NewClient(detail.Address, detail.AccessToken) - // 非文件夹情况下直接获取文件信息 - if !isDir { - if !isYaml(path) { - return ret, e.ErrPreloadServiceTemplate.AddDesc("File is not of type yaml or yml, select again") - } - fileInfo, err := ilyshinClient.GetFile(repoOwner, repoName, branchName, path) + base := path.Join(config.S3StoragePath(), repoName) + if exist, err := util.PathExists(base); !exist { + log.Warnf("path does not exist,err:%s", err) + err = command.RunGitCmds(detail, repoOwner, repoOwner, repoName, branchName, remoteName) if err != nil { - log.Errorf("Failed to get file info from ilyshin with path: %s, the error is %+v", path, err) - return ret, e.ErrPreloadServiceTemplate.AddDesc(err.Error()) + return nil, e.ErrPreloadServiceTemplate.AddDesc(err.Error()) } - extension := filepath.Ext(fileInfo.FileName) - fileName := fileInfo.FileName[0 : len(fileInfo.FileName)-len(extension)] - ret = append(ret, fileName) - return ret, nil } - treeInfo, err := ilyshinClient.ListTree(repoOwner, repoName, branchName, path) - if err != nil { - log.Errorf("Failed to get dir content from ilyshin with path: %s, the error is: %+v", path, err) - return ret, e.ErrPreloadServiceTemplate.AddDesc(err.Error()) - } - if isValidIlyshinServiceDir(treeInfo) { - svcName := path - if path == "" { - svcName = repoName + filePath := path.Join(base, loadPath) + + if !isDir { + if !isYaml(loadPath) { + log.Error("trying to preload a non-yaml file") + return nil, e.ErrPreloadServiceTemplate.AddDesc("Non-yaml service loading is not supported") } - pathList := strings.Split(svcName, "/") - folderName := pathList[len(pathList)-1] - ret = append(ret, folderName) - return ret, nil - } - isGrandparent := false - for _, entry := range treeInfo { - if entry.Type == "tree" { - subtreeInfo, err := ilyshinClient.ListTree(repoOwner, repoName, branchName, path) - if err != nil { - log.Errorf("Failed to get dir content from ilyshin with path: %s, the error is: %+v", path, err) - return ret, e.ErrPreloadServiceTemplate.AddDesc(err.Error()) + pathSegment := strings.Split(loadPath, "/") + fileName := pathSegment[len(pathSegment)-1] + ret = append(ret, getFileName(fileName)) + } else { + fileInfos, err := ioutil.ReadDir(filePath) + if err != nil { + log.Errorf("Failed to read directory info of path: %s, the error is: %s", filePath, err) + return nil, e.ErrPreloadServiceTemplate.AddDesc(err.Error()) + } + if isValidServiceDir(fileInfos) { + svcName := loadPath + if loadPath == "" { + svcName = repoName } - if isValidIlyshinServiceDir(subtreeInfo) { - isGrandparent = true - ret = append(ret, entry.Name) + pathList := strings.Split(svcName, "/") + folderName := pathList[len(pathList)-1] + ret = append(ret, folderName) + return ret, nil + } + isGrandParent := false + for _, file := range fileInfos { + if file.IsDir() { + subDirPath := fmt.Sprintf("%s/%s", filePath, file.Name()) + subtree, err := ioutil.ReadDir(subDirPath) + if err != nil { + log.Errorf("Failed to get subdir content from gitee with path: %s, the error is: %s", subDirPath, err) + return nil, e.ErrPreloadServiceTemplate.AddDesc(err.Error()) + } + if isValidServiceDir(subtree) { + ret = append(ret, getFileName(file.Name())) + isGrandParent = true + } } } + if !isGrandParent { + log.Errorf("invalid folder selected since no yaml is presented in path: %s", filePath) + return ret, e.ErrPreloadServiceTemplate.AddDesc("所选路径下没有yaml，请重新选择") + } } - if !isGrandparent { - log.Errorf("invalid folder selected since no yaml is presented in path: %s", path) - return ret, e.ErrPreloadServiceTemplate.AddDesc("所选路径下没有yaml，请重新选择") - } - return ret, nil } // 根据repo信息从gerrit加载服务 -func loadGerritService(username string, ch *poetry.CodeHost, repoOwner, repoName, branchName, remoteName string, args *LoadServiceReq, log *zap.SugaredLogger) error { +func loadGerritService(username string, ch *systemconfig.CodeHost, repoOwner, repoName, branchName, remoteName string, args *LoadServiceReq, force bool, log *zap.SugaredLogger) error { + if remoteName == "" { + remoteName = "origin" + } base := path.Join(config.S3StoragePath(), repoName) if _, err := os.Stat(base); os.IsNotExist(err) { - err = command.RunGitCmds(&codehost.Detail{Source: ch.Type, Address: ch.Address, OauthToken: ch.AccessToken}, repoOwner, repoName, branchName, remoteName) + err = command.RunGitCmds(ch, repoOwner, repoOwner, repoName, branchName, remoteName) if err != nil { return e.ErrLoadServiceTemplate.AddDesc(err.Error()) } } - gerritCli := gerrit.NewClient(ch.Address, ch.AccessToken) + gerritCli := gerrit.NewClient(ch.Address, ch.AccessToken, config.ProxyHTTPSAddr(), ch.EnableProxy) commit, err := gerritCli.GetCommitByBranch(repoName, branchName) if err != nil { log.Errorf("Failed to get latest commit info from repo: %s, the error is: %+v", repoName, err) @@ -341,6 +338,7 @@ func loadGerritService(username string, ch *poetry.CodeHost, repoOwner, repoName CodehostID: ch.ID, RepoName: repoName, RepoOwner: repoOwner, + RepoNamespace: repoOwner, // TODO gerrit need set namespace? BranchName: branchName, LoadPath: args.LoadPath, LoadFromDir: args.LoadFromDir, @@ -359,7 +357,7 @@ func loadGerritService(username string, ch *poetry.CodeHost, repoOwner, repoName Commit: commitInfo, Visibility: args.Visibility, } - _, err = CreateServiceTemplate(username, createSvcArgs, log) + _, err = CreateServiceTemplate(username, createSvcArgs, force, log) if err != nil { _, messageMap := e.ErrorMessage(err) if description, ok := messageMap["description"]; ok { @@ -374,8 +372,8 @@ func loadGerritService(username string, ch *poetry.CodeHost, repoOwner, repoName log.Errorf("Failed to read directory info of path: %s, the error is: %+v", filePath, err) return e.ErrLoadServiceTemplate.AddDesc(err.Error()) } - if isValidGerritServiceDir(fileInfos) { - return loadServiceFromGerrit(fileInfos, ch.ID, username, branchName, args.LoadPath, filePath, repoOwner, remoteName, repoName, args, commitInfo, log) + if isValidServiceDir(fileInfos) { + return loadServiceFromGerrit(fileInfos, ch.ID, username, branchName, args.LoadPath, filePath, repoOwner, remoteName, repoName, args, commitInfo, force, log) } for _, entry := range fileInfos { subtreeLoadPath := fmt.Sprintf("%s/%s", args.LoadPath, entry.Name()) @@ -385,8 +383,8 @@ func loadGerritService(username string, ch *poetry.CodeHost, repoOwner, repoName log.Errorf("Failed to read subdir info from gerrit package of path: %s, the error is: %+v", subtreePath, err) return e.ErrLoadServiceTemplate.AddDesc(err.Error()) } - if isValidGerritServiceDir(subtreeInfo) { - if err := loadServiceFromGerrit(subtreeInfo, ch.ID, username, branchName, subtreeLoadPath, subtreePath, repoOwner, remoteName, repoName, args, commitInfo, log); err != nil { + if isValidServiceDir(subtreeInfo) { + if err := loadServiceFromGerrit(subtreeInfo, ch.ID, username, branchName, subtreeLoadPath, subtreePath, repoOwner, remoteName, repoName, args, commitInfo, force, log); err != nil { return err } } @@ -394,12 +392,12 @@ func loadGerritService(username string, ch *poetry.CodeHost, repoOwner, repoName return nil } -func loadServiceFromGerrit(tree []os.FileInfo, id int, username, branchName, loadPath, path, repoOwner, remoteName, repoName string, args *LoadServiceReq, commit *models.Commit, log *zap.SugaredLogger) error { +func loadServiceFromGerrit(tree []os.FileInfo, id int, username, branchName, loadPath, path, repoOwner, remoteName, repoName string, args *LoadServiceReq, commit *models.Commit, force bool, log *zap.SugaredLogger) error { pathList := strings.Split(path, "/") var splittedYaml []string fileName := pathList[len(pathList)-1] serviceName := getFileName(fileName) - yamlList, err := extractGerritYamls(path, tree) + yamlList, err := extractYamls(path, tree) if err != nil { log.Errorf("Failed to extract yamls from gerrit package, the error is: %+v", err) return err @@ -431,7 +429,7 @@ func loadServiceFromGerrit(tree []os.FileInfo, id int, username, branchName, loa Visibility: args.Visibility, } - _, err = CreateServiceTemplate(username, createSvcArgs, log) + _, err = CreateServiceTemplate(username, createSvcArgs, force, log) if err != nil { _, messageMap := e.ErrorMessage(err) if description, ok := messageMap["description"]; ok { @@ -444,8 +442,8 @@ func loadServiceFromGerrit(tree []os.FileInfo, id int, username, branchName, loa } // load codehub service -func loadCodehubService(username string, ch *poetry.CodeHost, repoOwner, repoName, repoUUID, branchName string, args *LoadServiceReq, log *zap.SugaredLogger) error { - codeHubClient := codehub.NewCodeHubClient(ch.AccessKey, ch.SecretKey, ch.Region) +func loadCodehubService(username string, ch *systemconfig.CodeHost, repoOwner, repoName, repoUUID, branchName string, args *LoadServiceReq, force bool, log *zap.SugaredLogger) error { + codeHubClient := codehub.NewCodeHubClient(ch.AccessKey, ch.SecretKey, ch.Region, config.ProxyHTTPSAddr(), ch.EnableProxy) if !args.LoadFromDir { yamls, err := codeHubClient.GetYAMLContents(repoUUID, branchName, args.LoadPath, args.LoadFromDir, true) @@ -462,25 +460,26 @@ func loadCodehubService(username string, ch *poetry.CodeHost, repoOwner, repoNam srcPath := fmt.Sprintf("%s/%s/%s/blob/%s/%s", ch.Address, repoOwner, repoName, branchName, args.LoadPath) createSvcArgs := &models.Service{ - CodehostID: ch.ID, - RepoOwner: repoOwner, - RepoName: repoName, - RepoUUID: repoUUID, - BranchName: branchName, - SrcPath: srcPath, - LoadPath: args.LoadPath, - LoadFromDir: args.LoadFromDir, - KubeYamls: yamls, - CreateBy: username, - ServiceName: getFileName(args.LoadPath), - Type: args.Type, - ProductName: args.ProductName, - Source: ch.Type, - Yaml: util.CombineManifests(yamls), - Commit: &models.Commit{SHA: commit.ID, Message: commit.Message}, - Visibility: args.Visibility, - } - if _, err = CreateServiceTemplate(username, createSvcArgs, log); err != nil { + CodehostID: ch.ID, + RepoOwner: repoOwner, + RepoNamespace: repoOwner, // TODO codehub need fill namespace? + RepoName: repoName, + RepoUUID: repoUUID, + BranchName: branchName, + SrcPath: srcPath, + LoadPath: args.LoadPath, + LoadFromDir: args.LoadFromDir, + KubeYamls: yamls, + CreateBy: username, + ServiceName: getFileName(args.LoadPath), + Type: args.Type, + ProductName: args.ProductName, + Source: ch.Type, + Yaml: util.CombineManifests(yamls), + Commit: &models.Commit{SHA: commit.ID, Message: commit.Message}, + Visibility: args.Visibility, + } + if _, err = CreateServiceTemplate(username, createSvcArgs, force, log); err != nil { log.Errorf("Failed to create service template, serviceName:%s error: %s", createSvcArgs.ServiceName, err) _, messageMap := e.ErrorMessage(err) if description, ok := messageMap["description"]; ok { @@ -495,7 +494,7 @@ func loadCodehubService(username string, ch *poetry.CodeHost, repoOwner, repoNam return e.ErrLoadServiceTemplate.AddDesc(err.Error()) } if isValidCodehubServiceDir(treeNodes) { - return loadServiceFromCodehub(codeHubClient, treeNodes, ch, username, repoOwner, repoName, repoUUID, branchName, args.LoadPath, args, log) + return loadServiceFromCodehub(codeHubClient, treeNodes, ch, username, repoOwner, repoName, repoUUID, branchName, args.LoadPath, args, force, log) } for _, treeNode := range treeNodes { @@ -506,7 +505,7 @@ func loadCodehubService(username string, ch *poetry.CodeHost, repoOwner, repoNam return e.ErrLoadServiceTemplate.AddDesc(err.Error()) } if isValidCodehubServiceDir(subtree) { - if err := loadServiceFromCodehub(codeHubClient, subtree, ch, username, repoOwner, repoName, repoUUID, branchName, treeNode.Path, args, log); err != nil { + if err := loadServiceFromCodehub(codeHubClient, subtree, ch, username, repoOwner, repoName, repoUUID, branchName, treeNode.Path, args, force, log); err != nil { return err } } @@ -515,7 +514,7 @@ func loadCodehubService(username string, ch *poetry.CodeHost, repoOwner, repoNam return nil } -func loadServiceFromCodehub(client *codehub.CodeHubClient, tree []*codehub.TreeNode, ch *poetry.CodeHost, username, repoOwner, repoName, repoUUID, branchName, path string, args *LoadServiceReq, log *zap.SugaredLogger) error { +func loadServiceFromCodehub(client *codehub.CodeHubClient, tree []*codehub.TreeNode, ch *systemconfig.CodeHost, username, repoOwner, repoName, repoUUID, branchName, path string, args *LoadServiceReq, force bool, log *zap.SugaredLogger) error { pathList := strings.Split(path, "/") var splittedYaml []string serviceName := pathList[len(pathList)-1] @@ -558,7 +557,7 @@ func loadServiceFromCodehub(client *codehub.CodeHubClient, tree []*codehub.TreeN Visibility: args.Visibility, } - if _, err = CreateServiceTemplate(username, createSvcArgs, log); err != nil { + if _, err = CreateServiceTemplate(username, createSvcArgs, force, log); err != nil { _, messageMap := e.ErrorMessage(err) if description, ok := messageMap["description"]; ok { err = e.ErrLoadServiceTemplate.AddDesc(description.(string)) @@ -569,43 +568,61 @@ func loadServiceFromCodehub(client *codehub.CodeHubClient, tree []*codehub.TreeN return err } -// 根据 repo 信息从 Ilyshin 加载服务 -func loadIlyshinService(username string, ch *poetry.CodeHost, repoOwner, repoName, branchName string, args *LoadServiceReq, log *zap.SugaredLogger) error { - ilyshinClient := ilyshin.NewClient(ch.Address, ch.AccessToken) - repoInfo := fmt.Sprintf("%s/%s", repoOwner, repoName) - if !args.LoadFromDir { - file, err := ilyshinClient.GetFile(repoOwner, repoName, branchName, args.LoadPath) +// Load services from gitee based on repo information +func loadGiteeService(username string, ch *systemconfig.CodeHost, repoOwner, repoName, branchName, remoteName string, args *LoadServiceReq, force bool, log *zap.SugaredLogger) error { + if remoteName == "" { + remoteName = "origin" + } + base := path.Join(config.S3StoragePath(), repoName) + if _, err := os.Stat(base); os.IsNotExist(err) { + err = command.RunGitCmds(ch, repoOwner, repoName, repoName, branchName, remoteName) if err != nil { - log.Errorf("Failed to get file info for path: %s from ilyshin, the error is: %+v", args.LoadPath, err) return e.ErrLoadServiceTemplate.AddDesc(err.Error()) } - decodedContent, err := base64.StdEncoding.DecodeString(file.Content) + } + + giteeCli := gitee.NewClient(ch.ID, ch.AccessToken, config.ProxyHTTPSAddr(), ch.EnableProxy) + branch, err := giteeCli.GetSingleBranch(ch.AccessToken, repoOwner, repoName, branchName) + if err != nil { + log.Errorf("Failed to get latest commit info from repo: %s, the error is: %s", repoName, err) + return e.ErrLoadServiceTemplate.AddDesc(err.Error()) + } + commitInfo := &models.Commit{ + SHA: branch.Commit.Sha, + Message: branch.Commit.Commit.Message, + } + + filePath := path.Join(base, args.LoadPath) + if !args.LoadFromDir { + contentBytes, err := ioutil.ReadFile(path.Join(base, args.LoadPath)) if err != nil { - log.Errorf("Failed to decode file, the error is: %+v", err) + log.Errorf("Failed to read file of path: %s, the error is: %s", args.LoadPath, err) return e.ErrLoadServiceTemplate.AddDesc(err.Error()) } - srcPath := fmt.Sprintf("%s/%s/blob/%s/%s", ch.Address, repoInfo, branchName, args.LoadPath) - splittedYaml := SplitYaml(string(decodedContent)) + pathSegments := strings.Split(args.LoadPath, "/") + fileName := pathSegments[len(pathSegments)-1] + svcName := getFileName(fileName) + splittedYaml := SplitYaml(string(contentBytes)) + srcPath := fmt.Sprintf("%s/%s/%s/blob/%s/%s", ch.Address, repoOwner, repoName, branchName, args.LoadPath) createSvcArgs := &models.Service{ CodehostID: ch.ID, - RepoOwner: repoOwner, RepoName: repoName, + RepoOwner: repoOwner, BranchName: branchName, + SrcPath: srcPath, LoadPath: args.LoadPath, LoadFromDir: args.LoadFromDir, KubeYamls: splittedYaml, - SrcPath: srcPath, CreateBy: username, - ServiceName: getFileName(file.FileName), + ServiceName: svcName, Type: args.Type, ProductName: args.ProductName, - Source: setting.SourceFromIlyshin, - Yaml: string(decodedContent), - Commit: &models.Commit{SHA: file.CommitID}, + Source: setting.SourceFromGitee, + Yaml: string(contentBytes), + Commit: commitInfo, Visibility: args.Visibility, } - - _, err = CreateServiceTemplate(username, createSvcArgs, log) + _, err = CreateServiceTemplate(username, createSvcArgs, force, log) if err != nil { _, messageMap := e.ErrorMessage(err) if description, ok := messageMap["description"]; ok { @@ -615,67 +632,67 @@ func loadIlyshinService(username string, ch *poetry.CodeHost, repoOwner, repoNam } return nil } - - treeInfo, err := ilyshinClient.ListTree(repoOwner, repoName, branchName, args.LoadPath) + fileInfos, err := ioutil.ReadDir(filePath) if err != nil { - log.Errorf("Failed to get dir content from ilyshin with path: %s, the error is: %+v", args.LoadPath, err) + log.Errorf("Failed to read directory info of path: %s, the error is: %s", filePath, err) return e.ErrLoadServiceTemplate.AddDesc(err.Error()) } - if isValidIlyshinServiceDir(treeInfo) { - return loadServiceFromIlyshin(ilyshinClient, treeInfo, ch, username, repoOwner, repoName, branchName, args.LoadPath, args, log) + if isValidServiceDir(fileInfos) { + return loadServiceFromGitee(fileInfos, ch, username, branchName, args.LoadPath, filePath, repoOwner, remoteName, repoName, args, commitInfo, force, log) } - for _, treeNode := range treeInfo { - if treeNode.Type == "tree" { - subtree, err := ilyshinClient.ListTree(repoOwner, repoName, branchName, args.LoadPath) - if err != nil { - log.Errorf("Failed to get dir content from ilyshin with path: %s, the error is %+v", treeNode.Path, err) - return e.ErrLoadServiceTemplate.AddDesc(err.Error()) - } - if isValidIlyshinServiceDir(subtree) { - if err := loadServiceFromIlyshin(ilyshinClient, subtree, ch, username, repoOwner, repoName, branchName, treeNode.Path, args, log); err != nil { - return err - } + for _, entry := range fileInfos { + subtreeLoadPath := fmt.Sprintf("%s/%s", args.LoadPath, entry.Name()) + subtreePath := fmt.Sprintf("%s/%s", filePath, entry.Name()) + subtreeInfo, err := ioutil.ReadDir(subtreePath) + if err != nil { + log.Errorf("Failed to read subdir info from gitee package of path: %s, the error is: %s", subtreePath, err) + return e.ErrLoadServiceTemplate.AddDesc(err.Error()) + } + if isValidServiceDir(subtreeInfo) { + if err := loadServiceFromGitee(subtreeInfo, ch, username, branchName, subtreeLoadPath, subtreePath, repoOwner, remoteName, repoName, args, commitInfo, force, log); err != nil { + return err } } } return nil } -func loadServiceFromIlyshin(client *ilyshin.Client, tree []*ilyshin.TreeNode, ch *poetry.CodeHost, username, repoOwner, repoName, branchName, path string, args *LoadServiceReq, log *zap.SugaredLogger) error { +func loadServiceFromGitee(tree []os.FileInfo, ch *systemconfig.CodeHost, username, branchName, loadPath, path, repoOwner, remoteName, repoName string, args *LoadServiceReq, commit *models.Commit, force bool, log *zap.SugaredLogger) error { pathList := strings.Split(path, "/") - splittedYaml := []string{} - serviceName := pathList[len(pathList)-1] + var splittedYaml []string + fileName := pathList[len(pathList)-1] + serviceName := getFileName(fileName) repoInfo := fmt.Sprintf("%s/%s", repoOwner, repoName) - yamlList, sha, err := extractIlyshinYamls(client, tree, repoOwner, repoName, branchName) + yamlList, err := extractYamls(path, tree) if err != nil { - log.Errorf("Failed to extract yamls from ilyshin, the error is: %+v", err) - return e.ErrLoadServiceTemplate.AddDesc(err.Error()) + log.Errorf("Failed to extract yamls from gitee package, the error is: %+v", err) + return err } for _, yamlEntry := range yamlList { splittedYaml = append(splittedYaml, SplitYaml(yamlEntry)...) } yml := joinYamls(yamlList) - srcPath := fmt.Sprintf("%s/%s/tree/%s/%s", ch.Address, repoInfo, branchName, path) + srcPath := fmt.Sprintf("%s/%s/tree/%s/%s", ch.Address, repoInfo, branchName, loadPath) createSvcArgs := &models.Service{ CodehostID: ch.ID, - RepoOwner: repoOwner, - RepoName: repoName, BranchName: branchName, - LoadPath: path, + RepoName: repoName, + RepoOwner: repoOwner, + LoadPath: loadPath, + SrcPath: srcPath, LoadFromDir: args.LoadFromDir, KubeYamls: splittedYaml, - SrcPath: srcPath, CreateBy: username, ServiceName: serviceName, Type: args.Type, ProductName: args.ProductName, - Source: setting.SourceFromIlyshin, + Source: setting.SourceFromGitee, Yaml: yml, - Commit: &models.Commit{SHA: sha}, + Commit: commit, Visibility: args.Visibility, } - _, err = CreateServiceTemplate(username, createSvcArgs, log) + _, err = CreateServiceTemplate(username, createSvcArgs, force, log) if err != nil { _, messageMap := e.ErrorMessage(err) if description, ok := messageMap["description"]; ok { @@ -688,7 +705,7 @@ func loadServiceFromIlyshin(client *ilyshin.Client, tree []*ilyshin.TreeNode, ch } // 根据github repo决定服务是否可以更新这个repo地址 -func validateServiceUpdateGithub(detail *poetry.CodeHost, serviceName, repoOwner, repoName, branchName, path string, isDir bool) error { +func validateServiceUpdateGithub(detail *systemconfig.CodeHost, serviceName, repoOwner, repoName, branchName, path string, isDir bool) error { ctx := context.Background() tokenSource := oauth2.StaticTokenSource( &oauth2.Token{AccessToken: detail.AccessToken}, @@ -735,7 +752,7 @@ func validateServiceUpdateGithub(detail *poetry.CodeHost, serviceName, repoOwner } // 根据gitlab repo决定服务是否可以更新这个repo地址 -func validateServiceUpdateGitlab(detail *poetry.CodeHost, serviceName, repoOwner, repoName, branchName, path string, isDir bool) error { +func validateServiceUpdateGitlab(detail *systemconfig.CodeHost, serviceName, repoOwner, repoName, branchName, path string, isDir bool) error { repoInfo := fmt.Sprintf("%s/%s", repoOwner, repoName) gitlabClient, err := gitlab.NewOAuthClient(detail.AccessToken, gitlab.WithBaseURL(detail.Address)) @@ -787,18 +804,13 @@ func validateServiceUpdateGitlab(detail *poetry.CodeHost, serviceName, repoOwner } // 根据gerrit repo决定服务是否可以更新这个repo地址 -func validateServiceUpdateGerrit(detail *poetry.CodeHost, serviceName, repoName, branchName, remoteName, loadPath string, isDir bool) error { +func validateServiceUpdateGerrit(detail *systemconfig.CodeHost, serviceName, repoName, branchName, remoteName, loadPath string, isDir bool) error { + if remoteName == "" { + remoteName = "origin" + } base := path.Join(config.S3StoragePath(), repoName) if _, err := os.Stat(base); os.IsNotExist(err) { - chDetail := &codehost.Detail{ - ID: detail.ID, - Name: "", - Address: detail.Address, - Owner: detail.Namespace, - Source: detail.Type, - OauthToken: detail.AccessToken, - } - err = command.RunGitCmds(chDetail, setting.GerritDefaultOwner, repoName, branchName, remoteName) + err = command.RunGitCmds(detail, setting.GerritDefaultOwner, setting.GerritDefaultOwner, repoName, branchName, remoteName) if err != nil { return e.ErrValidateServiceUpdate.AddDesc(err.Error()) } @@ -807,7 +819,7 @@ func validateServiceUpdateGerrit(detail *poetry.CodeHost, serviceName, repoName, filePath := path.Join(base, loadPath) if !isDir { if !isYaml(loadPath) { - log.Errorf("trying to preload a non-yaml file") + log.Error("trying to preload a non-yaml file") return e.ErrPreloadServiceTemplate.AddDesc("Non-yaml service loading is not supported") } pathSegment := strings.Split(loadPath, "/") @@ -823,7 +835,7 @@ func validateServiceUpdateGerrit(detail *poetry.CodeHost, serviceName, repoName, log.Errorf("Failed to read directory info of path: %s, the error is: %+v", filePath, err) return e.ErrValidateServiceUpdate.AddDesc(err.Error()) } - if isValidGerritServiceDir(fileInfos) { + if isValidServiceDir(fileInfos) { svcName := loadPath if loadPath == "" { svcName = repoName @@ -839,8 +851,8 @@ func validateServiceUpdateGerrit(detail *poetry.CodeHost, serviceName, repoName, return e.ErrValidateServiceUpdate.AddDesc("所选路径中没有yaml，请重新选择") } -func validateServiceUpdateCodehub(detail *poetry.CodeHost, serviceName, repoName, repoUUID, branchName, loadPath string, isDir bool) error { - codeHubClient := codehub.NewCodeHubClient(detail.AccessKey, detail.SecretKey, detail.Region) +func validateServiceUpdateCodehub(detail *systemconfig.CodeHost, serviceName, repoName, repoUUID, branchName, loadPath string, isDir bool) error { + codeHubClient := codehub.NewCodeHubClient(detail.AccessKey, detail.SecretKey, detail.Region, config.ProxyHTTPSAddr(), detail.EnableProxy) // 非文件夹情况下直接获取文件信息 if !isDir { if !isYaml(loadPath) { @@ -879,34 +891,42 @@ func validateServiceUpdateCodehub(detail *poetry.CodeHost, serviceName, repoName return e.ErrValidateServiceUpdate.AddDesc("所选路径中没有yaml，请重新选择") } -// 根据 ilyshin repo 决定服务是否可以更新这个 repo 地址 -func validateServiceUpdateIlyshin(detail *poetry.CodeHost, serviceName, repoOwner, repoName, branchName, path string, isDir bool) error { - ilyshinClient := ilyshin.NewClient(detail.Address, detail.AccessToken) - // 非文件夹情况下直接获取文件信息 - if !isDir { - if !isYaml(path) { - return e.ErrValidateServiceUpdate.AddDesc("File is not of type yaml or yml, select again") - } - fileInfo, err := ilyshinClient.GetFile(repoOwner, repoName, branchName, path) +// Determine whether the service can update the repo address according to the gitee repo +func validateServiceUpdateGitee(detail *systemconfig.CodeHost, serviceName, repoOwner, repoName, branchName, remoteName, loadPath string, isDir bool) error { + if remoteName == "" { + remoteName = "origin" + } + base := path.Join(config.S3StoragePath(), repoName) + if exist, err := util.PathExists(base); !exist { + log.Warnf("path does not exist,err:%s", err) + err := command.RunGitCmds(detail, repoOwner, repoOwner, repoName, branchName, remoteName) if err != nil { - log.Errorf("Failed to get file info from ilyshin with path: %s, the error is %+v", path, err) return e.ErrValidateServiceUpdate.AddDesc(err.Error()) } - if getFileName(fileInfo.FileName) != serviceName { - log.Errorf("The loaded file name [%s] is the same as the service to be updated: [%s]", fileInfo.FileName, serviceName) + } + + filePath := path.Join(base, loadPath) + if !isDir { + if !isYaml(loadPath) { + log.Error("trying to preload a non-yaml file") + return e.ErrPreloadServiceTemplate.AddDesc("Non-yaml service loading is not supported") + } + pathSegment := strings.Split(loadPath, "/") + fileName := pathSegment[len(pathSegment)-1] + if getFileName(fileName) != serviceName { + log.Errorf("The loaded file name [%s] is the same as the service to be updated: [%s]", fileName, serviceName) return e.ErrValidateServiceUpdate.AddDesc("文件名称和服务名称不一致") } return nil } - - treeInfo, err := ilyshinClient.ListTree(repoOwner, repoName, branchName, path) + fileInfos, err := ioutil.ReadDir(filePath) if err != nil { - log.Errorf("Failed to get dir content from gitlab with path: %s, the error is: %+v", path, err) + log.Errorf("Failed to read directory info of path: %s, the error is: %+v", filePath, err) return e.ErrValidateServiceUpdate.AddDesc(err.Error()) } - if isValidIlyshinServiceDir(treeInfo) { - svcName := path - if path == "" { + if isValidServiceDir(fileInfos) { + svcName := loadPath + if loadPath == "" { svcName = repoName } pathList := strings.Split(svcName, "/") @@ -938,16 +958,7 @@ func isValidGitlabServiceDir(child []*gitlab.TreeNode) bool { return false } -func isValidIlyshinServiceDir(child []*ilyshin.TreeNode) bool { - for _, entry := range child { - if entry.Type == "blob" && isYaml(entry.Name) { - return true - } - } - return false -} - -func isValidGerritServiceDir(child []os.FileInfo) bool { +func isValidServiceDir(child []os.FileInfo) bool { for _, file := range child { if !file.IsDir() && isYaml(file.Name()) { return true @@ -985,7 +996,7 @@ func extractCodehubYamls(client *codehub.CodeHubClient, tree []*codehub.TreeNode return ret, nil } -func extractGerritYamls(basePath string, tree []os.FileInfo) ([]string, error) { +func extractYamls(basePath string, tree []os.FileInfo) ([]string, error) { var ret []string for _, entry := range tree { if !entry.IsDir() && isYaml(entry.Name()) { @@ -999,25 +1010,3 @@ func extractGerritYamls(basePath string, tree []os.FileInfo) ([]string, error) { } return ret, nil } - -func extractIlyshinYamls(client *ilyshin.Client, tree []*ilyshin.TreeNode, repoOwner, repoName, branchName string) ([]string, string, error) { - ret := []string{} - var sha string - for _, entry := range tree { - if isYaml(entry.Name) { - fileInfo, err := client.GetFile(repoOwner, repoName, branchName, entry.Path) - if err != nil { - log.Errorf("Failed to download ilyshin file: %s, the error is: %+v", entry.Path, err) - return nil, "", err - } - decodedFile, err := base64.StdEncoding.DecodeString(fileInfo.Content) - if err != nil { - log.Errorf("Failed to decode content from the given file of path: %s, the error is: %s", entry.Path, err) - return nil, "", err - } - ret = append(ret, string(decodedFile)) - sha = fileInfo.CommitID - } - } - return ret, sha, nil -} diff --git a/pkg/microservice/aslan/core/service/service/new_loader.go b/pkg/microservice/aslan/core/service/service/new_loader.go index 3c2a05765058ae3d81f814b134ea93bf8b56b8bd..58b71c0918f27eafdd6fa80bbc11c38647d292c5 100644 --- a/pkg/microservice/aslan/core/service/service/new_loader.go +++ b/pkg/microservice/aslan/core/service/service/new_loader.go @@ -30,13 +30,13 @@ import ( githubservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/github" gitlabservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/gitlab" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/poetry" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/util" ) -func preloadService(ch *poetry.CodeHost, owner, repo, branch, path string, isDir bool, logger *zap.SugaredLogger) ([]string, error) { +func preloadService(ch *systemconfig.CodeHost, owner, repo, branch, path string, isDir bool, logger *zap.SugaredLogger) ([]string, error) { logger.Infof("Preloading service from %s with owner %s, repo %s, branch %s and path %s", ch.Type, owner, repo, branch, path) loader, err := getLoader(ch) @@ -95,8 +95,8 @@ type serviceInfo struct { yamls []string } -func loadService(username string, ch *poetry.CodeHost, owner, repo, branch string, args *LoadServiceReq, logger *zap.SugaredLogger) error { - logger.Infof("Loading service from %s with owner %s, repo %s, branch %s and path %s", ch.Type, owner, repo, branch, args.LoadPath) +func loadService(username string, ch *systemconfig.CodeHost, owner, namespace, repo, branch string, args *LoadServiceReq, force bool, logger *zap.SugaredLogger) error { + logger.Infof("Loading service from %s with owner %s, namespace %s, repo %s, branch %s and path %s", ch.Type, owner, namespace, repo, branch, args.LoadPath) project, err := templaterepo.NewProductColl().Find(args.ProductName) if err != nil { @@ -112,7 +112,7 @@ func loadService(username string, ch *poetry.CodeHost, owner, repo, branch strin var services []serviceInfo if !args.LoadFromDir { - yamls, err := loader.GetYAMLContents(owner, repo, args.LoadPath, branch, false, true) + yamls, err := loader.GetYAMLContents(namespace, repo, args.LoadPath, branch, false, true) if err != nil { logger.Errorf("Failed to get yamls under path %s, err: %s", args.LoadPath, err) return e.ErrLoadServiceTemplate.AddDesc(err.Error()) @@ -120,7 +120,7 @@ func loadService(username string, ch *poetry.CodeHost, owner, repo, branch strin services = []serviceInfo{{path: args.LoadPath, isDir: false, yamls: yamls}} } else { - treeNodes, err := loader.GetTree(owner, repo, args.LoadPath, branch) + treeNodes, err := loader.GetTree(namespace, repo, args.LoadPath, branch) if err != nil { logger.Errorf("Failed to get tree under path %s, err: %s", args.LoadPath, err) return e.ErrLoadServiceTemplate.AddDesc(err.Error()) @@ -133,7 +133,7 @@ func loadService(username string, ch *poetry.CodeHost, owner, repo, branch strin if len(files) > 0 { var yamls []string for _, f := range files { - res, err := loader.GetYAMLContents(owner, repo, f.FullPath, branch, false, true) + res, err := loader.GetYAMLContents(namespace, repo, f.FullPath, branch, false, true) if err != nil { logger.Errorf("Failed to get yamls under path %s, err: %s", f.FullPath, err) return e.ErrLoadServiceTemplate.AddDesc(err.Error()) @@ -144,7 +144,7 @@ func loadService(username string, ch *poetry.CodeHost, owner, repo, branch strin services = []serviceInfo{{path: args.LoadPath, isDir: true, yamls: yamls}} } else if len(folders) > 0 { for _, f := range folders { - res, err := loader.GetYAMLContents(owner, repo, f.FullPath, branch, true, true) + res, err := loader.GetYAMLContents(namespace, repo, f.FullPath, branch, true, true) if err != nil { logger.Errorf("Failed to get yamls under path %s, err: %s", f.FullPath, err) return e.ErrLoadServiceTemplate.AddDesc(err.Error()) @@ -165,7 +165,7 @@ func loadService(username string, ch *poetry.CodeHost, owner, repo, branch strin return e.ErrInvalidParam.AddDesc(fmt.Sprintf("A service with same name %s is already existing", serviceName)) } - commit, err := loader.GetLatestRepositoryCommit(owner, repo, info.path, branch) + commit, err := loader.GetLatestRepositoryCommit(namespace, repo, info.path, branch) if err != nil { logger.Errorf("Failed to get latest commit under path %s, error: %s", info.path, err) return e.ErrLoadServiceTemplate.AddDesc(err.Error()) @@ -176,24 +176,25 @@ func loadService(username string, ch *poetry.CodeHost, owner, repo, branch strin pathType = "blob" } createSvcArgs := &models.Service{ - CodehostID: ch.ID, - RepoName: repo, - RepoOwner: owner, - BranchName: branch, - LoadPath: info.path, - LoadFromDir: info.isDir, - KubeYamls: info.yamls, - SrcPath: fmt.Sprintf("%s/%s/%s/%s/%s/%s", ch.Address, owner, repo, pathType, branch, info.path), - CreateBy: username, - ServiceName: serviceName, - Type: args.Type, - ProductName: args.ProductName, - Source: ch.Type, - Yaml: util.CombineManifests(info.yamls), - Commit: &models.Commit{SHA: commit.SHA, Message: commit.Message}, - Visibility: args.Visibility, + CodehostID: ch.ID, + RepoName: repo, + RepoOwner: owner, + RepoNamespace: namespace, + BranchName: branch, + LoadPath: info.path, + LoadFromDir: info.isDir, + KubeYamls: info.yamls, + SrcPath: fmt.Sprintf("%s/%s/%s/%s/%s/%s", ch.Address, namespace, repo, pathType, branch, info.path), + CreateBy: username, + ServiceName: serviceName, + Type: args.Type, + ProductName: args.ProductName, + Source: ch.Type, + Yaml: util.CombineManifests(info.yamls), + Commit: &models.Commit{SHA: commit.SHA, Message: commit.Message}, + Visibility: args.Visibility, } - _, err = CreateServiceTemplate(username, createSvcArgs, logger) + _, err = CreateServiceTemplate(username, createSvcArgs, force, logger) if err != nil { logger.Errorf("Failed to create service template, err: %s", err) _, messageMap := e.ErrorMessage(err) @@ -236,12 +237,12 @@ type yamlLoader interface { GetTree(owner, repo, path, branch string) ([]*git.TreeNode, error) } -func getLoader(ch *poetry.CodeHost) (yamlLoader, error) { +func getLoader(ch *systemconfig.CodeHost) (yamlLoader, error) { switch ch.Type { case setting.SourceFromGithub: - return githubservice.NewClient(ch.AccessToken, config.ProxyHTTPSAddr()), nil + return githubservice.NewClient(ch.AccessToken, config.ProxyHTTPSAddr(), ch.EnableProxy), nil case setting.SourceFromGitlab: - return gitlabservice.NewClient(ch.Address, ch.AccessToken) + return gitlabservice.NewClient(ch.ID, ch.Address, ch.AccessToken, config.ProxyHTTPSAddr(), ch.EnableProxy) default: // should not have happened here log.DPanicf("invalid source: %s", ch.Type) diff --git a/pkg/microservice/aslan/core/service/service/pm.go b/pkg/microservice/aslan/core/service/service/pm.go index a41b714613e4987344bf2f49cd8ef090f848d00b..4b2012afd6f6eda06103c5bd1f3f65a155d5ff88 100644 --- a/pkg/microservice/aslan/core/service/service/pm.go +++ b/pkg/microservice/aslan/core/service/service/pm.go @@ -28,6 +28,8 @@ import ( commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/pm" + commonutil "github.com/koderover/zadig/pkg/microservice/aslan/core/common/util" "github.com/koderover/zadig/pkg/setting" e "github.com/koderover/zadig/pkg/tool/errors" ) @@ -44,6 +46,9 @@ func CreatePMService(username string, args *ServiceTmplBuildObject, log *zap.Sug if !config.ServiceNameRegex.MatchString(args.ServiceTmplObject.ServiceName) { return e.ErrInvalidParam.AddDesc("服务名称格式错误，请检查") } + if err := commonutil.CheckDefineResourceParam(args.Build.PreBuild.ResReq, args.Build.PreBuild.ResReqSpec); err != nil { + return e.ErrInvalidParam.AddDesc(err.Error()) + } opt := &commonrepo.ServiceFindOption{ ServiceName: args.ServiceTmplObject.ServiceName, @@ -70,7 +75,11 @@ func CreatePMService(username string, args *ServiceTmplBuildObject, log *zap.Sug if err := commonrepo.NewServiceColl().Delete(args.ServiceTmplObject.ServiceName, args.ServiceTmplObject.Type, args.ServiceTmplObject.ProductName, setting.ProductStatusDeleting, args.ServiceTmplObject.Revision); err != nil { log.Errorf("pmService.delete %s error: %v", args.ServiceTmplObject.ServiceName, err) } - + envStatus, err := pm.GenerateEnvStatus(args.ServiceTmplObject.EnvConfigs, log) + if err != nil { + log.Errorf("GenerateEnvStatus %s", err) + return err + } serviceObj := &commonmodels.Service{ ServiceName: args.ServiceTmplObject.ServiceName, Type: args.ServiceTmplObject.Type, @@ -79,6 +88,7 @@ func CreatePMService(username string, args *ServiceTmplBuildObject, log *zap.Sug Visibility: args.ServiceTmplObject.Visibility, HealthChecks: args.ServiceTmplObject.HealthChecks, EnvConfigs: args.ServiceTmplObject.EnvConfigs, + EnvStatuses: envStatus, CreateTime: time.Now().Unix(), CreateBy: username, BuildName: args.Build.Name, @@ -90,7 +100,7 @@ func CreatePMService(username string, args *ServiceTmplBuildObject, log *zap.Sug } // Confirm whether the build exists - build, err := commonrepo.NewBuildColl().Find(&commonrepo.BuildFindOption{Name: args.Build.Name}) + build, err := commonrepo.NewBuildColl().Find(&commonrepo.BuildFindOption{Name: args.Build.Name, ProductName: args.Build.ProductName}) if err != nil { if err := commonservice.CreateBuild(username, args.Build, log); err != nil { log.Errorf("pmService.Create build %s error: %v", args.Build.Name, err) diff --git a/pkg/microservice/aslan/core/service/service/service.go b/pkg/microservice/aslan/core/service/service/service.go index c3b23dfa5a3a18e2e5fb6275d75083901322d59b..96b6970dd8f49c5958237a524f6ee9326d0779dd 100644 --- a/pkg/microservice/aslan/core/service/service/service.go +++ b/pkg/microservice/aslan/core/service/service/service.go @@ -24,6 +24,7 @@ import ( "io/ioutil" "os" "path" + "regexp" "strconv" "strings" "sync" @@ -35,7 +36,6 @@ import ( configbase "github.com/koderover/zadig/pkg/config" "github.com/koderover/zadig/pkg/microservice/aslan/config" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" templatemodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" @@ -46,7 +46,8 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/codehost" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/gerrit" "github.com/koderover/zadig/pkg/tool/httpclient" @@ -56,21 +57,23 @@ import ( ) type ServiceOption struct { - ServiceModules []*ServiceModule `json:"service_module"` - SystemVariable []*Variable `json:"system_variable"` - CustomVariable []*templatemodels.RenderKV `json:"custom_variable"` - Yaml string `json:"yaml"` - Service *commonmodels.Service `json:"service,omitempty"` + ServiceModules []*ServiceModule `json:"service_module"` + SystemVariable []*Variable `json:"system_variable"` + CustomVariable []*templatemodels.RenderKV `json:"custom_variable"` + TemplateVariable []*Variable `json:"template_variable"` + Yaml string `json:"yaml"` + Service *commonmodels.Service `json:"service,omitempty"` } type ServiceModule struct { *commonmodels.Container - BuildName string `json:"build_name"` + BuildNames []string `json:"build_names"` } type Variable struct { - Key string `json:"key"` - Value string `json:"value"` + Key string `json:"key"` + Value string `json:"value"` + Description string `json:"description,omitempty"` } type YamlPreviewForPorts struct { @@ -134,10 +137,22 @@ type YamlValidatorReq struct { Yaml string `json:"yaml,omitempty"` } +type YamlViewServiceTemplateReq struct { + ServiceName string `json:"service_name"` + ProjectName string `json:"project_name"` + EnvName string `json:"env_name"` + Variables []*templatemodels.RenderKV `json:"variables"` +} + +type ReleaseNamingRule struct { + NamingRule string `json:"naming"` + ServiceName string `json:"service_name"` +} + func ListServicesInExtenalEnv(tmpResp *commonservice.ServiceTmplResp, log *zap.SugaredLogger) { opt := &commonrepo.ProductListOptions{ Source: setting.SourceFromExternal, - ExcludeStatus: setting.ProductStatusDeleting, + ExcludeStatus: []string{setting.ProductStatusDeleting, setting.ProductStatusUnknown}, } products, err := commonrepo.NewProductColl().List(opt) if err != nil { @@ -176,6 +191,24 @@ func GetServiceTemplateOption(serviceName, productName string, revision int64, l return serviceOption, err } +func GetTemplateVariables(args *commonmodels.Service) []*Variable { + if args.TemplateID == "" { + return nil + } + templateInfo, err := commonrepo.NewYamlTemplateColl().GetById(args.TemplateID) + if err != nil { + log.Errorf("failed to find template with id: %s for service: %s, err: %s", args.TemplateID, args.ServiceName, err) + return nil + } + + variables, err := buildYamlTemplateVariables(args, templateInfo) + if err != nil { + log.Errorf("failed to extract template variables for service: %s, err: %s", args.ServiceName, err) + return nil + } + return variables +} + func GetServiceOption(args *commonmodels.Service, log *zap.SugaredLogger) (*ServiceOption, error) { serviceOption := new(ServiceOption) @@ -183,10 +216,17 @@ func GetServiceOption(args *commonmodels.Service, log *zap.SugaredLogger) (*Serv for _, container := range args.Containers { serviceModule := new(ServiceModule) serviceModule.Container = container - buildObj, _ := commonrepo.NewBuildColl().Find(&commonrepo.BuildFindOption{ProductName: args.ProductName, ServiceName: args.ServiceName, Targets: []string{container.Name}}) - if buildObj != nil { - serviceModule.BuildName = buildObj.Name + serviceModule.ImageName = util.GetImageNameFromContainerInfo(container.ImageName, container.Name) + buildObjs, err := commonrepo.NewBuildColl().List(&commonrepo.BuildListOption{ProductName: args.ProductName, ServiceName: args.ServiceName, Targets: []string{container.Name}}) + if err != nil { + return nil, err + } + + buildNames := sets.NewString() + for _, buildObj := range buildObjs { + buildNames.Insert(buildObj.Name) } + serviceModule.BuildNames = buildNames.List() serviceModules = append(serviceModules, serviceModule) } serviceOption.ServiceModules = serviceModules @@ -204,6 +244,7 @@ func GetServiceOption(args *commonmodels.Service, log *zap.SugaredLogger) (*Serv Key: "$EnvName$", Value: ""}, } + serviceOption.TemplateVariable = GetTemplateVariables(args) renderKVs, err := commonservice.ListServicesRenderKeys([]*templatemodels.ServiceInfo{{Name: args.ServiceName, Owner: args.ProductName}}, log) if err != nil { log.Errorf("ListServicesRenderKeys %s error: %v", args.ServiceName, err) @@ -248,54 +289,72 @@ func GetServiceOption(args *commonmodels.Service, log *zap.SugaredLogger) (*Serv } if args.Source == setting.SourceFromGitlab || args.Source == setting.SourceFromGithub || - args.Source == setting.SourceFromGerrit || args.Source == setting.SourceFromCodeHub || - args.Source == setting.SourceFromIlyshin { + args.Source == setting.SourceFromGerrit || args.Source == setting.SourceFromCodeHub || args.Source == setting.SourceFromGitee { serviceOption.Yaml = args.Yaml } return serviceOption, nil } -func CreateK8sWorkLoads(ctx context.Context, requestID, username string, productName string, workLoads []models.Workload, clusterID, namespace string, envName string, log *zap.SugaredLogger) error { - kubeClient, err := kube.GetKubeClient(clusterID) +type K8sWorkloadsArgs struct { + WorkLoads []commonmodels.Workload `json:"workLoads"` + EnvName string `json:"env_name"` + ClusterID string `json:"cluster_id"` + Namespace string `json:"namespace"` + ProductName string `json:"product_name"` + RegistryID string `json:"registry_id"` +} + +func CreateK8sWorkLoads(ctx context.Context, requestID, userName string, args *K8sWorkloadsArgs, log *zap.SugaredLogger) error { + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), args.ClusterID) if err != nil { - log.Errorf("[%s] error: %v", namespace, err) + log.Errorf("[%s] error: %v", args.Namespace, err) return err } // 检查环境是否存在，envName和productName唯一 - opt := &commonrepo.ProductFindOptions{Name: productName, EnvName: envName} + opt := &commonrepo.ProductFindOptions{Name: args.ProductName, EnvName: args.EnvName} if _, err := commonrepo.NewProductColl().Find(opt); err == nil { - log.Errorf("[%s][P:%s] duplicate envName in the same product", envName, productName) + log.Errorf("[%s][P:%s] duplicate envName in the same project", args.EnvName, args.ProductName) return e.ErrCreateEnv.AddDesc(e.DuplicateEnvErrMsg) } // todo Add data filter var ( - workloadsTmp []models.Workload + workloadsTmp []commonmodels.Workload mu sync.Mutex ) - // pre judge workLoads same name serviceString := sets.NewString() - services, _ := commonrepo.NewServiceColl().ListExternalWorkloadsBy(productName, "") + services, _ := commonrepo.NewServiceColl().ListExternalWorkloadsBy(args.ProductName, "") for _, v := range services { serviceString.Insert(v.ServiceName) } - for _, workload := range workLoads { - if serviceString.Has(workload.Name) { - return e.ErrCreateTemplate.AddDesc(fmt.Sprintf("do not support import same service name: %s", workload.Name)) - } - } + //for _, workload := range workLoads { + // if serviceString.Has(workload.Name) { + // return e.ErrCreateTemplate.AddDesc(fmt.Sprintf("do not support import same service name: %s", workload.Name)) + // } + //} g := new(errgroup.Group) - for _, workload := range workLoads { + for _, workload := range args.WorkLoads { tempWorkload := workload g.Go(func() error { + // If the service is already included in the database service template, add it to the new association table + if serviceString.Has(tempWorkload.Name) { + return commonrepo.NewServicesInExternalEnvColl().Create(&commonmodels.ServicesInExternalEnv{ + ProductName: args.ProductName, + ServiceName: tempWorkload.Name, + EnvName: args.EnvName, + Namespace: args.Namespace, + ClusterID: args.ClusterID, + }) + } + var bs []byte switch tempWorkload.Type { case setting.Deployment: - bs, _, err = getter.GetDeploymentYaml(namespace, tempWorkload.Name, kubeClient) + bs, _, err = getter.GetDeploymentYaml(args.Namespace, tempWorkload.Name, kubeClient) case setting.StatefulSet: - bs, _, err = getter.GetStatefulSetYaml(namespace, tempWorkload.Name, kubeClient) + bs, _, err = getter.GetStatefulSetYaml(args.Namespace, tempWorkload.Name, kubeClient) } if len(bs) == 0 || err != nil { @@ -305,22 +364,22 @@ func CreateK8sWorkLoads(ctx context.Context, requestID, username string, product mu.Lock() defer mu.Unlock() - workloadsTmp = append(workloadsTmp, models.Workload{ - EnvName: envName, + workloadsTmp = append(workloadsTmp, commonmodels.Workload{ + EnvName: args.EnvName, Name: tempWorkload.Name, Type: tempWorkload.Type, - ProductName: productName, + ProductName: args.ProductName, }) - return CreateWorkloadTemplate(username, &models.Service{ + return CreateWorkloadTemplate(userName, &commonmodels.Service{ ServiceName: tempWorkload.Name, Yaml: string(bs), - ProductName: productName, - CreateBy: username, + ProductName: args.ProductName, + CreateBy: userName, Type: setting.K8SDeployType, WorkloadType: tempWorkload.Type, Source: setting.SourceFromExternal, - EnvName: envName, + EnvName: args.EnvName, Revision: 1, }, log) }) @@ -331,31 +390,33 @@ func CreateK8sWorkLoads(ctx context.Context, requestID, username string, product // 没有环境，创建环境 if _, err = commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ - Name: productName, - EnvName: envName, + Name: args.ProductName, + EnvName: args.EnvName, }); err != nil { - if err := service.CreateProduct(username, requestID, &commonmodels.Product{ - ProductName: productName, + if err := service.CreateProduct(userName, requestID, &commonmodels.Product{ + ProductName: args.ProductName, Source: setting.SourceFromExternal, - ClusterID: clusterID, - EnvName: envName, - Namespace: namespace, + ClusterID: args.ClusterID, + RegistryID: args.RegistryID, + EnvName: args.EnvName, + Namespace: args.Namespace, + UpdateBy: userName, }, log); err != nil { return e.ErrCreateProduct.AddDesc("create product Error for unknown reason") } } - workLoadStat, err := commonrepo.NewWorkLoadsStatColl().Find(clusterID, namespace) + workLoadStat, err := commonrepo.NewWorkLoadsStatColl().Find(args.ClusterID, args.Namespace) if err != nil { workLoadStat = &commonmodels.WorkloadStat{ - ClusterID: clusterID, - Namespace: namespace, + ClusterID: args.ClusterID, + Namespace: args.Namespace, Workloads: workloadsTmp, } return commonrepo.NewWorkLoadsStatColl().Create(workLoadStat) } - workLoadStat.Workloads = replaceWorkloads(workLoadStat.Workloads, workloadsTmp, envName) + workLoadStat.Workloads = replaceWorkloads(workLoadStat.Workloads, workloadsTmp, args.EnvName) return commonrepo.NewWorkLoadsStatColl().UpdateWorkloads(workLoadStat) } @@ -374,7 +435,7 @@ type UpdateWorkloadsArgs struct { } func UpdateWorkloads(ctx context.Context, requestID, username, productName, envName string, args UpdateWorkloadsArgs, log *zap.SugaredLogger) error { - kubeClient, err := kube.GetKubeClient(args.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), args.ClusterID) if err != nil { log.Errorf("[%s] error: %s", args.Namespace, err) return err @@ -384,6 +445,19 @@ func UpdateWorkloads(ctx context.Context, requestID, username, productName, envN log.Errorf("[%s][%s]NewWorkLoadsStatColl().Find %s", args.ClusterID, args.Namespace, err) return err } + externalEnvServices, _ := commonrepo.NewServicesInExternalEnvColl().List(&commonrepo.ServicesInExternalEnvArgs{ + ProductName: productName, + EnvName: envName, + }) + + for _, externalEnvService := range externalEnvServices { + workloadStat.Workloads = append(workloadStat.Workloads, commonmodels.Workload{ + ProductName: externalEnvService.ProductName, + EnvName: externalEnvService.EnvName, + Name: externalEnvService.ServiceName, + }) + } + diff := map[string]*ServiceWorkloadsUpdateAction{} originSet := sets.NewString() uploadSet := sets.NewString() @@ -392,6 +466,7 @@ func UpdateWorkloads(ctx context.Context, requestID, username, productName, envN originSet.Insert(v.Name) } } + for _, v := range args.WorkLoads { uploadSet.Insert(v.Name) } @@ -412,13 +487,6 @@ func UpdateWorkloads(ctx context.Context, requestID, username, productName, envN } } } - // pre judge workLoads same name - services, _ := commonrepo.NewServiceColl().ListExternalWorkloadsBy(productName, "") - for _, v := range services { - if addString.Has(v.ServiceName) { - return e.ErrCreateTemplate.AddDesc(fmt.Sprintf("do not support import same service name: %s", v.ServiceName)) - } - } for _, v := range args.WorkLoads { if addString.Has(v.Name) { @@ -432,13 +500,46 @@ func UpdateWorkloads(ctx context.Context, requestID, username, productName, envN } } + otherExternalEnvServices, err := commonrepo.NewServicesInExternalEnvColl().List(&commonrepo.ServicesInExternalEnvArgs{ + ProductName: productName, + ExcludeEnvName: envName, + }) + if err != nil { + log.Errorf("failed to list external service, error:%s", err) + } + + externalEnvServiceM := make(map[string]*commonmodels.ServicesInExternalEnv) + for _, externalEnvService := range otherExternalEnvServices { + externalEnvServiceM[externalEnvService.ServiceName] = externalEnvService + } for _, v := range diff { switch v.Operation { // 删除workload的引用 case "delete": - err = commonrepo.NewServiceColl().UpdateExternalServicesStatus(v.Name, productName, setting.ProductStatusDeleting, envName) - if err != nil { - log.Errorf("UpdateStatus external services error:%s", err) + if externalService, isExist := externalEnvServiceM[v.Name]; !isExist { + if err = commonrepo.NewServiceColl().UpdateExternalServicesStatus(v.Name, productName, setting.ProductStatusDeleting, envName); err != nil { + log.Errorf("UpdateStatus external services error:%s", err) + } + } else { + // Update the env name in the service + if err = commonrepo.NewServiceColl().UpdateExternalServiceEnvName(v.Name, productName, externalService.EnvName); err != nil { + log.Errorf("UpdateEnvName external services error:%s", err) + } + // Delete the reference in the original service + if err = commonrepo.NewServicesInExternalEnvColl().Delete(&commonrepo.ServicesInExternalEnvArgs{ + ProductName: externalService.ProductName, + EnvName: externalService.EnvName, + ServiceName: externalService.ServiceName, + }); err != nil { + log.Errorf("delete service in external env envName:%s error:%s", externalService.EnvName, err) + } + } + if err = commonrepo.NewServicesInExternalEnvColl().Delete(&commonrepo.ServicesInExternalEnvArgs{ + ProductName: productName, + EnvName: envName, + ServiceName: v.Name, + }); err != nil { + log.Errorf("delete service in external env envName:%s error:%s", envName, err) } // 添加workload的引用 case "add": @@ -454,7 +555,7 @@ func UpdateWorkloads(ctx context.Context, requestID, username, productName, envN delete(diff, v.Name) continue } - if err = CreateWorkloadTemplate(username, &models.Service{ + if err = CreateWorkloadTemplate(username, &commonmodels.Service{ ServiceName: v.Name, Yaml: string(bs), ProductName: productName, @@ -476,15 +577,15 @@ func UpdateWorkloads(ctx context.Context, requestID, username, productName, envN return commonrepo.NewWorkLoadsStatColl().UpdateWorkloads(workloadStat) } -func updateWorkloads(existWorkloads []models.Workload, diff map[string]*ServiceWorkloadsUpdateAction, envName string, productName string) (result []models.Workload) { - existWorkloadsMap := map[string]models.Workload{} +func updateWorkloads(existWorkloads []commonmodels.Workload, diff map[string]*ServiceWorkloadsUpdateAction, envName string, productName string) (result []commonmodels.Workload) { + existWorkloadsMap := map[string]commonmodels.Workload{} for _, v := range existWorkloads { existWorkloadsMap[v.Name] = v } for _, v := range diff { switch v.Operation { case "add": - vv := models.Workload{ + vv := commonmodels.Workload{ EnvName: envName, Name: v.Name, Type: v.Type, @@ -501,9 +602,9 @@ func updateWorkloads(existWorkloads []models.Workload, diff map[string]*ServiceW return result } -func replaceWorkloads(existWorkloads []models.Workload, newWorkloads []models.Workload, envName string) []models.Workload { - var result []models.Workload - workloadMap := map[string]models.Workload{} +func replaceWorkloads(existWorkloads []commonmodels.Workload, newWorkloads []commonmodels.Workload, envName string) []commonmodels.Workload { + var result []commonmodels.Workload + workloadMap := map[string]commonmodels.Workload{} for _, workload := range existWorkloads { if workload.EnvName != envName { workloadMap[workload.Name] = workload @@ -544,7 +645,7 @@ func CreateWorkloadTemplate(userName string, args *commonmodels.Service, log *za //获取项目里面的所有服务 if len(productTempl.Services) > 0 && !sets.NewString(productTempl.Services[0]...).Has(args.ServiceName) { productTempl.Services[0] = append(productTempl.Services[0], args.ServiceName) - } else { + } else if len(productTempl.Services) == 0 { productTempl.Services = [][]string{{args.ServiceName}} } //更新项目模板 @@ -555,7 +656,21 @@ func CreateWorkloadTemplate(userName string, args *commonmodels.Service, log *za } } } else { - return e.ErrCreateTemplate.AddDesc("do not support import same service name") + product, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: args.ProductName, + EnvName: args.EnvName, + }) + if err != nil { + return err + } + return commonrepo.NewServicesInExternalEnvColl().Create(&commonmodels.ServicesInExternalEnv{ + ProductName: args.ProductName, + ServiceName: args.ServiceName, + EnvName: args.EnvName, + Namespace: product.Namespace, + ClusterID: product.ClusterID, + }) + //return e.ErrCreateTemplate.AddDesc("do not support import same service name") } if err := commonrepo.NewServiceColl().Delete(args.ServiceName, args.Type, args.ProductName, setting.ProductStatusDeleting, 0); err != nil { @@ -569,7 +684,7 @@ func CreateWorkloadTemplate(userName string, args *commonmodels.Service, log *za return nil } -func CreateServiceTemplate(userName string, args *commonmodels.Service, log *zap.SugaredLogger) (*ServiceOption, error) { +func CreateServiceTemplate(userName string, args *commonmodels.Service, force bool, log *zap.SugaredLogger) (*ServiceOption, error) { opt := &commonrepo.ServiceFindOption{ ServiceName: args.ServiceName, ProductName: args.ProductName, @@ -587,52 +702,53 @@ func CreateServiceTemplate(userName string, args *commonmodels.Service, log *zap // 在更新数据库前检查是否有完全重复的Item，如果有，则退出。 serviceTmpl, notFoundErr := commonrepo.NewServiceColl().Find(opt) - if notFoundErr == nil { - if args.Type == setting.K8SDeployType && args.Source == serviceTmpl.Source { - // 配置来源为zadig，对比配置内容是否变化，需要对比Yaml内容 - // 如果Source没有设置，默认认为是zadig平台管理配置方式 - if args.Source == setting.SourceFromZadig || args.Source == "" { - if args.Yaml != "" && serviceTmpl.Yaml == args.Yaml { - log.Info("Yaml config remains the same, quit creation.") - return GetServiceOption(serviceTmpl, log) - } - } - // 配置来源为Gitlab，对比配置的ChangeLog是否变化 - if args.Source == setting.SourceFromGitlab || args.Source == setting.SourceFromGithub || args.Source == setting.SourceFromCodeHub || args.Source == setting.SourceFromIlyshin { - if args.Commit != nil && serviceTmpl.Commit != nil && args.Commit.SHA == serviceTmpl.Commit.SHA { - log.Infof("%s change log remains the same, quit creation", args.Source) - return GetServiceOption(serviceTmpl, log) - } - if args.LoadPath != serviceTmpl.LoadPath && serviceTmpl.LoadPath != "" { - log.Errorf("Changing load path is not allowed") - return nil, e.ErrCreateTemplate.AddDesc("不允许更改加载路径") - } - } else if args.Source == setting.SourceFromGerrit { - if args.Yaml != "" && serviceTmpl.Yaml == args.Yaml { - log.Info("gerrit Yaml config remains the same, quit creation.") - return GetServiceOption(serviceTmpl, log) - } - if args.LoadPath != serviceTmpl.LoadPath && serviceTmpl.LoadPath != "" { - log.Errorf("Changing load path is not allowed") - return nil, e.ErrCreateTemplate.AddDesc("不允许更改加载路径") - } - if err := updateGerritWebhookByService(serviceTmpl, args); err != nil { - log.Infof("gerrit update webhook err :%v", err) - return nil, err - } - } - } else if args.Source == setting.SourceFromGerrit { - err := GetGerritServiceYaml(args, log) - if err != nil { - log.Errorf("GetGerritServiceYaml from gerrit failed, error: %v", err) - return nil, err - } - //创建gerrit webhook - if err = createGerritWebhookByService(args.GerritCodeHostID, args.ServiceName, args.GerritRepoName, args.GerritBranchName); err != nil { - log.Errorf("createGerritWebhookByService error: %v", err) - return nil, err - } - } + if notFoundErr == nil && !force { + return nil, fmt.Errorf("service:%s already exists", serviceTmpl.ServiceName) + //if args.Type == setting.K8SDeployType && args.Source == serviceTmpl.Source { + // // 配置来源为zadig，对比配置内容是否变化，需要对比Yaml内容 + // // 如果Source没有设置，默认认为是zadig平台管理配置方式 + // if args.Source == setting.SourceFromZadig || args.Source == "" { + // if args.Yaml != "" && serviceTmpl.Yaml == args.Yaml { + // log.Info("Yaml config remains the same, quit creation.") + // return GetServiceOption(serviceTmpl, log) + // } + // } + // // 配置来源为Gitlab，对比配置的ChangeLog是否变化 + // if args.Source == setting.SourceFromGitlab || args.Source == setting.SourceFromGithub || args.Source == setting.SourceFromCodeHub || args.Source == setting.SourceFromGitee { + // if args.Commit != nil && serviceTmpl.Commit != nil && args.Commit.SHA == serviceTmpl.Commit.SHA { + // log.Infof("%s change log remains the same, quit creation", args.Source) + // return GetServiceOption(serviceTmpl, log) + // } + // if args.LoadPath != serviceTmpl.LoadPath && serviceTmpl.LoadPath != "" { + // log.Errorf("Changing load path is not allowed") + // return nil, e.ErrCreateTemplate.AddDesc("不允许更改加载路径") + // } + // } else if args.Source == setting.SourceFromGerrit { + // if args.Yaml != "" && serviceTmpl.Yaml == args.Yaml { + // log.Info("gerrit Yaml config remains the same, quit creation.") + // return GetServiceOption(serviceTmpl, log) + // } + // if args.LoadPath != serviceTmpl.LoadPath && serviceTmpl.LoadPath != "" { + // log.Errorf("Changing load path is not allowed") + // return nil, e.ErrCreateTemplate.AddDesc("不允许更改加载路径") + // } + // if err := updateGerritWebhookByService(serviceTmpl, args); err != nil { + // log.Infof("gerrit update webhook err :%v", err) + // return nil, err + // } + // } + //} else if args.Source == setting.SourceFromGerrit { + // err := GetGerritServiceYaml(args, log) + // if err != nil { + // log.Errorf("GetGerritServiceYaml from gerrit failed, error: %v", err) + // return nil, err + // } + // //创建gerrit webhook + // if err = createGerritWebhookByService(args.GerritCodeHostID, args.ServiceName, args.GerritRepoName, args.GerritBranchName); err != nil { + // log.Errorf("createGerritWebhookByService error: %v", err) + // return nil, err + // } + //} } else { if args.Source == setting.SourceFromGerrit { //创建gerrit webhook @@ -676,17 +792,22 @@ func CreateServiceTemplate(userName string, args *commonmodels.Service, log *zap } commonservice.ProcessServiceWebhook(args, serviceTmpl, args.ServiceName, log) + err = service.AutoDeployYamlServiceToEnvs(userName, "", args, log) + if err != nil { + return nil, e.ErrCreateTemplate.AddErr(err) + } + return GetServiceOption(args, log) } -func UpdateServiceTemplate(args *commonservice.ServiceTmplObject) error { +func UpdateServiceVisibility(args *commonservice.ServiceTmplObject) error { currentService, err := commonrepo.NewServiceColl().Find(&commonrepo.ServiceFindOption{ ProductName: args.ProductName, ServiceName: args.ServiceName, Revision: args.Revision, }) if err != nil { - log.Errorf("Can not find service with option %+v", args) + log.Errorf("Can not find service with option %+v. Error: %s", args, err) return err } @@ -710,7 +831,7 @@ func UpdateServiceTemplate(args *commonservice.ServiceTmplObject) error { } envStatuses := make([]*commonmodels.EnvStatus, 0) - // 去掉检查状态中不存在的环境和主机 + // Remove environments and hosts that do not exist in the check status for _, envStatus := range args.EnvStatuses { var existEnv, existHost bool @@ -748,6 +869,83 @@ func UpdateServiceTemplate(args *commonservice.ServiceTmplObject) error { return commonrepo.NewServiceColl().Update(updateArgs) } +func UpdateServiceHealthCheckStatus(args *commonservice.ServiceTmplObject) error { + currentService, err := commonrepo.NewServiceColl().Find(&commonrepo.ServiceFindOption{ + ProductName: args.ProductName, + ServiceName: args.ServiceName, + Revision: args.Revision, + }) + if err != nil { + log.Errorf("Can not find service with option %+v. Error: %s", args, err) + return err + } + var changeEnvStatus []*commonmodels.EnvStatus + var changeEnvConfigs []*commonmodels.EnvConfig + + changeEnvConfigs = append(changeEnvConfigs, args.EnvConfigs...) + envConfigsSet := sets.String{} + for _, v := range changeEnvConfigs { + envConfigsSet.Insert(v.EnvName) + } + for _, v := range currentService.EnvConfigs { + if !envConfigsSet.Has(v.EnvName) { + changeEnvConfigs = append(changeEnvConfigs, v) + } + } + var privateKeys []*commonmodels.PrivateKey + for _, envConfig := range args.EnvConfigs { + privateKeys, err = commonrepo.NewPrivateKeyColl().ListHostIPByArgs(&commonrepo.ListHostIPArgs{IDs: envConfig.HostIDs}) + if err != nil { + log.Errorf("ListNameByArgs ids err:%s", err) + return err + } + + privateKeysByLabels, err := commonrepo.NewPrivateKeyColl().ListHostIPByArgs(&commonrepo.ListHostIPArgs{Labels: envConfig.Labels}) + if err != nil { + log.Errorf("ListNameByArgs labels err:%s", err) + return err + } + privateKeys = append(privateKeys, privateKeysByLabels...) + } + privateKeysSet := sets.NewString() + for _, v := range privateKeys { + tmp := commonmodels.EnvStatus{ + HostID: v.ID.Hex(), + EnvName: args.EnvName, + Address: v.IP, + } + if !privateKeysSet.Has(tmp.HostID) { + changeEnvStatus = append(changeEnvStatus, &tmp) + privateKeysSet.Insert(tmp.HostID) + } + } + // get env status + for _, v := range currentService.EnvStatuses { + if v.EnvName != args.EnvName { + changeEnvStatus = append(changeEnvStatus, v) + } + } + // generate env status for this env + updateArgs := &commonmodels.Service{ + ProductName: args.ProductName, + ServiceName: args.ServiceName, + Visibility: args.Visibility, + Revision: args.Revision, + Type: args.Type, + CreateBy: args.Username, + EnvConfigs: changeEnvConfigs, + EnvStatuses: changeEnvStatus, + } + return commonrepo.NewServiceColl().UpdateServiceHealthCheckStatus(updateArgs) +} + +func extractHostIPs(privateKeys []*commonmodels.PrivateKey, ips sets.String) sets.String { + for _, privateKey := range privateKeys { + ips.Insert(privateKey.IP) + } + return ips +} + func YamlValidator(args *YamlValidatorReq) []string { //validateWithCacheMap := make(map[string]*gojsonschema.Schema) errorDetails := make([]string, 0) @@ -767,14 +965,138 @@ func YamlValidator(args *YamlValidatorReq) []string { yamlData = config.ServiceNameAlias.ReplaceAllLiteralString(yamlData, args.ServiceName) if err := yaml.Unmarshal([]byte(yamlData), &resKind); err != nil { - log.Errorf("yaml unmarsh err: %s", err) - errorDetails = append(errorDetails, "Invalid yaml format. The content must be a series of valid Kubernetes resources") + errorDetails = append(errorDetails, fmt.Sprintf("Invalid yaml format. The content must be a series of valid Kubernetes resources. err: %s", err)) } } } return errorDetails } +func YamlViewServiceTemplate(args *YamlViewServiceTemplateReq) (string, error) { + opt := &commonrepo.ServiceFindOption{ + ServiceName: args.ServiceName, + ProductName: args.ProjectName, + ExcludeStatus: setting.ProductStatusDeleting, + } + svcTmpl, err := commonrepo.NewServiceColl().Find(opt) + if err != nil { + return "", err + } + + renderSet := new(commonmodels.RenderSet) + renderSet.KVs = args.Variables + parsedYaml := commonservice.RenderValueForString(svcTmpl.Yaml, renderSet) + if args.EnvName != "" { + prod, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: args.ProjectName, + EnvName: args.EnvName, + }) + if err != nil { + return "", err + } + + parsedYaml = kube.ParseSysKeys(prod.Namespace, prod.EnvName, prod.ProductName, args.ServiceName, parsedYaml) + cerSvc := prod.GetServiceMap() + svcInfo, found := cerSvc[args.ServiceName] + if found { + parsedYaml, err = replaceContainerImages(parsedYaml, svcTmpl.Containers, svcInfo.Containers) + if err != nil { + return "", err + } + } + } + + return parsedYaml, nil +} + +func replaceContainerImages(tmpl string, ori []*commonmodels.Container, replace []*commonmodels.Container) (string, error) { + replaceMap := make(map[string]string) + for _, container := range replace { + replaceMap[container.Name] = container.Image + } + + for _, container := range ori { + imageRex, err := regexp.Compile("image:\\s*" + container.Image) + if err != nil { + return "", err + } + if _, ok := replaceMap[container.Name]; !ok { + continue + } + tmpl = imageRex.ReplaceAllLiteralString(tmpl, fmt.Sprintf("image: %s", replaceMap[container.Name])) + } + + return tmpl, nil +} + +func UpdateReleaseNamingRule(userName, requestID, projectName string, args *ReleaseNamingRule, log *zap.SugaredLogger) error { + serviceTemplate, err := commonrepo.NewServiceColl().Find(&commonrepo.ServiceFindOption{ + ServiceName: args.ServiceName, + Revision: 0, + Type: setting.HelmDeployType, + ProductName: projectName, + ExcludeStatus: setting.ProductStatusDeleting, + }) + if err != nil { + return err + } + + // check if namings rule changes for services deployed in envs + if serviceTemplate.GetReleaseNaming() == args.NamingRule { + products, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{ + Name: projectName, + }) + if err != nil { + return fmt.Errorf("failed to list envs for product: %s, err: %s", projectName, err) + } + modified := false + for _, product := range products { + if pSvc, ok := product.GetServiceMap()[args.ServiceName]; ok && pSvc.Revision != serviceTemplate.Revision { + modified = true + break + } + } + if !modified { + return nil + } + } + + serviceTemplate.ReleaseNaming = args.NamingRule + rev, err := getNextServiceRevision(projectName, args.ServiceName) + if err != nil { + return fmt.Errorf("failed to get service next revision, service %s, err: %s", args.ServiceName, err) + } + + basePath := config.LocalServicePath(serviceTemplate.ProductName, serviceTemplate.ServiceName) + if err = commonservice.PreLoadServiceManifests(basePath, serviceTemplate); err != nil { + return fmt.Errorf("failed to load chart info for service %s, err: %s", serviceTemplate.ServiceName, err) + } + + fsTree := os.DirFS(config.LocalServicePath(projectName, serviceTemplate.ServiceName)) + s3Base := config.ObjectStorageServicePath(projectName, serviceTemplate.ServiceName) + err = fs.ArchiveAndUploadFilesToS3(fsTree, []string{fmt.Sprintf("%s-%d", serviceTemplate.ServiceName, rev)}, s3Base, log) + if err != nil { + return fmt.Errorf("failed to upload chart info for service %s, err: %s", serviceTemplate.ServiceName, err) + } + + serviceTemplate.Revision = rev + err = commonrepo.NewServiceColl().Create(serviceTemplate) + if err != nil { + return fmt.Errorf("failed to update relase naming for service: %s, err: %s", args.ServiceName, err) + } + + go func() { + // reinstall services in envs + err = service.ReInstallHelmSvcInAllEnvs(projectName, serviceTemplate) + if err != nil { + title := fmt.Sprintf("服务 [%s] 重建失败", args.ServiceName) + commonservice.SendErrorMessage(userName, title, requestID, err, log) + } + }() + + return nil +} + func DeleteServiceTemplate(serviceName, serviceType, productName, isEnvTemplate, visibility string, log *zap.SugaredLogger) error { openEnvTemplate, _ := strconv.ParseBool(isEnvTemplate) if openEnvTemplate && visibility == "public" { @@ -845,13 +1167,13 @@ func DeleteServiceTemplate(serviceName, serviceType, productName, isEnvTemplate, if serviceType == setting.HelmDeployType { // 更新helm renderset - err = removeServiceFromRenderset(productName, productName, serviceName) + err = removeServiceFromRenderset(productName, productName, "", serviceName) if err != nil { log.Warnf("failed to update renderset: %s when deleting service: %s, err: %s", productName, serviceName, err.Error()) } // 把该服务相关的s3的数据从仓库删除 - if err = fs.DeleteArchivedFileFromS3(serviceName, configbase.ObjectStorageServicePath(productName, serviceName), log); err != nil { + if err = fs.DeleteArchivedFileFromS3([]string{serviceName}, configbase.ObjectStorageServicePath(productName, serviceName), log); err != nil { log.Warnf("Failed to delete file %s, err: %s", serviceName, err) } //onBoarding流程时，需要删除预定的renderset中的已经存在的renderchart信息 @@ -879,7 +1201,7 @@ func DeleteServiceTemplate(serviceName, serviceType, productName, isEnvTemplate, envNames := []string{"dev", "qa"} for _, envName := range envNames { rendersetName := commonservice.GetProductEnvNamespace(envName, productName, "") - err := removeServiceFromRenderset(productName, rendersetName, serviceName) + err := removeServiceFromRenderset(productName, rendersetName, envName, serviceName) if err != nil { log.Warnf("failed to update renderset: %s when deleting service: %s, err: %s", rendersetName, serviceName, err.Error()) } @@ -893,8 +1215,8 @@ func DeleteServiceTemplate(serviceName, serviceType, productName, isEnvTemplate, } // remove specific services from rendersets.chartinfos -func removeServiceFromRenderset(productName, renderName, serviceName string) error { - renderOpt := &commonrepo.RenderSetFindOption{Name: renderName} +func removeServiceFromRenderset(productName, renderName, envName, serviceName string) error { + renderOpt := &commonrepo.RenderSetFindOption{Name: renderName, ProductTmpl: productName, EnvName: envName} if rs, err := commonrepo.NewRenderSetColl().Find(renderOpt); err == nil { chartInfos := make([]*templatemodels.RenderChart, 0) for _, chartInfo := range rs.ChartInfos { @@ -1002,7 +1324,7 @@ func GetGerritServiceYaml(args *commonmodels.Service, log *zap.SugaredLogger) er base := path.Join(config.S3StoragePath(), args.GerritRepoName) if _, err := os.Stat(base); os.IsNotExist(err) { - err = command.RunGitCmds(codehostDetail, setting.GerritDefaultOwner, args.GerritRepoName, args.GerritBranchName, args.GerritRemoteName) + err = command.RunGitCmds(codehostDetail, setting.GerritDefaultOwner, setting.GerritDefaultOwner, args.GerritRepoName, args.GerritBranchName, args.GerritRemoteName) if err != nil { return err } @@ -1050,7 +1372,7 @@ func ensureServiceTmpl(userName string, args *commonmodels.Service, log *zap.Sug args.Containers = make([]*commonmodels.Container, 0) } // Only the gerrit/spock/external type needs to be processed by yaml - if args.Source == setting.SourceFromGerrit || args.Source == setting.SourceFromZadig || args.Source == setting.SourceFromExternal { + if args.Source == setting.SourceFromGerrit || args.Source == setting.SourceFromZadig || args.Source == setting.SourceFromExternal || args.Source == setting.ServiceSourceTemplate || args.Source == setting.SourceFromGitee { // 拆分 all-in-one yaml文件 // 替换分隔符 args.Yaml = util.ReplaceWrapLine(args.Yaml) @@ -1103,18 +1425,23 @@ func setCurrentContainerImages(args *commonmodels.Service) error { var srvContainers []*commonmodels.Container for _, data := range args.KubeYamls { yamlDataArray := SplitYaml(data) - for _, yamlData := range yamlDataArray { + for index, yamlData := range yamlDataArray { resKind := new(KubeResourceKind) //在Unmarshal之前填充渲染变量{{.}} yamlData = config.RenderTemplateAlias.ReplaceAllLiteralString(yamlData, "ssssssss") // replace $Service$ with service name yamlData = config.ServiceNameAlias.ReplaceAllLiteralString(yamlData, args.ServiceName) + // replace $Product$ with product name + yamlData = config.ProductNameAlias.ReplaceAllLiteralString(yamlData, args.ProductName) if err := yaml.Unmarshal([]byte(yamlData), &resKind); err != nil { return fmt.Errorf("unmarshal ResourceKind error: %v", err) } if resKind == nil { + if index == 0 { + continue + } return errors.New("nil ReourceKind") } @@ -1169,8 +1496,9 @@ func getContainers(data string) ([]*commonmodels.Container, error) { } container := &commonmodels.Container{ - Name: nameStr, - Image: imageStr, + Name: nameStr, + Image: imageStr, + ImageName: util.ExtractImageName(imageStr), } containers = append(containers, container) @@ -1208,8 +1536,9 @@ func getCronJobContainers(data string) ([]*commonmodels.Container, error) { } container := &commonmodels.Container{ - Name: nameStr, - Image: imageStr, + Name: nameStr, + Image: imageStr, + ImageName: util.ExtractImageName(imageStr), } containers = append(containers, container) @@ -1219,15 +1548,15 @@ func getCronJobContainers(data string) ([]*commonmodels.Container, error) { } func updateGerritWebhookByService(lastService, currentService *commonmodels.Service) error { - var codehostDetail *codehost.Detail + var codehostDetail *systemconfig.CodeHost var err error if lastService != nil && lastService.Source == setting.SourceFromGerrit { - codehostDetail, err = codehost.GetCodehostDetail(lastService.GerritCodeHostID) + codehostDetail, err = systemconfig.New().GetCodeHost(lastService.GerritCodeHostID) if err != nil { log.Errorf("updateGerritWebhookByService GetCodehostDetail err:%v", err) return err } - cl := gerrit.NewHTTPClient(codehostDetail.Address, codehostDetail.OauthToken) + cl := gerrit.NewHTTPClient(codehostDetail.Address, codehostDetail.AccessToken) webhookURLPrefix := fmt.Sprintf("/%s/%s/%s", "a/config/server/webhooks~projects", gerrit.Escape(lastService.GerritRepoName), "remotes") _, _ = cl.Delete(fmt.Sprintf("%s/%s", webhookURLPrefix, gerrit.RemoteName)) _, err = cl.Delete(fmt.Sprintf("%s/%s", webhookURLPrefix, lastService.ServiceName)) @@ -1235,14 +1564,14 @@ func updateGerritWebhookByService(lastService, currentService *commonmodels.Serv log.Errorf("updateGerritWebhookByService deleteGerritWebhook err:%v", err) } } - var detail *codehost.Detail + var detail *systemconfig.CodeHost if lastService.GerritCodeHostID == currentService.GerritCodeHostID && codehostDetail != nil { detail = codehostDetail } else { - detail, _ = codehost.GetCodehostDetail(currentService.GerritCodeHostID) + detail, _ = systemconfig.New().GetCodeHost(currentService.GerritCodeHostID) } - cl := gerrit.NewHTTPClient(detail.Address, detail.OauthToken) + cl := gerrit.NewHTTPClient(detail.Address, detail.AccessToken) webhookURL := fmt.Sprintf("/%s/%s/%s/%s", "a/config/server/webhooks~projects", gerrit.Escape(currentService.GerritRepoName), "remotes", currentService.ServiceName) if _, err := cl.Get(webhookURL); err != nil { log.Errorf("updateGerritWebhookByService getGerritWebhook err:%v", err) @@ -1262,13 +1591,13 @@ func updateGerritWebhookByService(lastService, currentService *commonmodels.Serv } func createGerritWebhookByService(codehostID int, serviceName, repoName, branchName string) error { - detail, err := codehost.GetCodehostDetail(codehostID) + detail, err := systemconfig.New().GetCodeHost(codehostID) if err != nil { log.Errorf("createGerritWebhookByService GetCodehostDetail err:%v", err) return err } - cl := gerrit.NewHTTPClient(detail.Address, detail.OauthToken) + cl := gerrit.NewHTTPClient(detail.Address, detail.AccessToken) webhookURL := fmt.Sprintf("/%s/%s/%s/%s", "a/config/server/webhooks~projects", gerrit.Escape(repoName), "remotes", serviceName) if _, err := cl.Get(webhookURL); err != nil { log.Errorf("createGerritWebhookByService getGerritWebhook err:%v", err) @@ -1287,7 +1616,7 @@ func createGerritWebhookByService(codehostID int, serviceName, repoName, branchN return nil } -func syncGerritLatestCommit(service *commonmodels.Service) (*codehost.Detail, error) { +func syncGerritLatestCommit(service *commonmodels.Service) (*systemconfig.CodeHost, error) { if service.GerritCodeHostID == 0 { return nil, fmt.Errorf("codehostId不能是空的") } @@ -1297,9 +1626,9 @@ func syncGerritLatestCommit(service *commonmodels.Service) (*codehost.Detail, er if service.GerritBranchName == "" { return nil, fmt.Errorf("branchName不能是空的") } - ch, _ := codehost.GetCodehostDetail(service.GerritCodeHostID) + ch, _ := systemconfig.New().GetCodeHost(service.GerritCodeHostID) - gerritCli := gerrit.NewClient(ch.Address, ch.OauthToken) + gerritCli := gerrit.NewClient(ch.Address, ch.AccessToken, config.ProxyHTTPSAddr(), ch.EnableProxy) commit, err := gerritCli.GetCommitByBranch(service.GerritRepoName, service.GerritBranchName) if err != nil { return nil, err diff --git a/pkg/microservice/aslan/core/service/service/template.go b/pkg/microservice/aslan/core/service/service/template.go new file mode 100644 index 0000000000000000000000000000000000000000..b2a27caf2dd2308e80eb545d78a0fe2c19a729b8 --- /dev/null +++ b/pkg/microservice/aslan/core/service/service/template.go @@ -0,0 +1,355 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + + "go.uber.org/zap" + + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/log" +) + +type LoadServiceFromYamlTemplateReq struct { + ServiceName string `json:"service_name"` + ProjectName string `json:"project_name"` + TemplateID string `json:"template_id"` + AutoSync bool `json:"auto_sync"` + Variables []*Variable `json:"variables"` +} + +func geneCreateFromDetail(templateId string, variables []*Variable) *commonmodels.CreateFromYamlTemplate { + vs := make([]*commonmodels.Variable, 0, len(variables)) + for _, kv := range variables { + vs = append(vs, &commonmodels.Variable{ + Key: kv.Key, + Value: kv.Value, + }) + } + return &commonmodels.CreateFromYamlTemplate{ + TemplateID: templateId, + Variables: vs, + } +} + +func LoadServiceFromYamlTemplate(username string, req *LoadServiceFromYamlTemplateReq, force bool, logger *zap.SugaredLogger) error { + projectName, serviceName, templateID, variables, autoSync := req.ProjectName, req.ServiceName, req.TemplateID, req.Variables, req.AutoSync + template, err := commonrepo.NewYamlTemplateColl().GetById(templateID) + if err != nil { + logger.Errorf("Failed to find template of ID: %s, the error is: %s", templateID, err) + return err + } + renderedYaml := renderYamlFromTemplate(template.Content, projectName, serviceName, variables) + service := &commonmodels.Service{ + ServiceName: serviceName, + Type: setting.K8SDeployType, + ProductName: projectName, + Source: setting.ServiceSourceTemplate, + Yaml: renderedYaml, + Visibility: setting.PrivateVisibility, + TemplateID: templateID, + AutoSync: autoSync, + CreateFrom: geneCreateFromDetail(templateID, variables), + } + _, err = CreateServiceTemplate(username, service, force, logger) + if err != nil { + logger.Errorf("Failed to create service template from template ID: %s, the error is: %s", templateID, err) + } + return err +} + +func ReloadServiceFromYamlTemplate(username string, req *LoadServiceFromYamlTemplateReq, logger *zap.SugaredLogger) error { + projectName, serviceName, templateID, variables, autoSync := req.ProjectName, req.ServiceName, req.TemplateID, req.Variables, req.AutoSync + service, err := commonrepo.NewServiceColl().Find(&commonrepo.ServiceFindOption{ + ServiceName: serviceName, + ProductName: projectName, + }) + if err != nil { + logger.Errorf("Cannot find service of name [%s] from project [%s], the error is: %s", serviceName, projectName, err) + return err + } + if service.Source != setting.ServiceSourceTemplate { + return errors.New("service is not created from template") + } + if service.TemplateID == "" { + return fmt.Errorf("failed to find template id for service: %s", serviceName) + } + template, err := commonrepo.NewYamlTemplateColl().GetById(templateID) + if err != nil { + logger.Errorf("Failed to find template of ID: %s, the error is: %s", templateID, err) + return err + } + renderedYaml := renderYamlFromTemplate(template.Content, projectName, serviceName, variables) + svc := &commonmodels.Service{ + ServiceName: serviceName, + Type: setting.K8SDeployType, + ProductName: projectName, + Source: setting.ServiceSourceTemplate, + Yaml: renderedYaml, + Visibility: setting.PrivateVisibility, + TemplateID: templateID, + AutoSync: autoSync, + CreateFrom: geneCreateFromDetail(templateID, variables), + } + _, err = CreateServiceTemplate(username, svc, true, logger) + if err != nil { + logger.Errorf("Failed to create service template from template ID: %s, the error is: %s", service.TemplateID, err) + } + return err +} + +func renderYamlFromTemplate(yaml, productName, serviceName string, variables []*Variable) string { + for _, variable := range variables { + yaml = strings.ReplaceAll(yaml, buildVariable(variable.Key), variable.Value) + } + yaml = strings.ReplaceAll(yaml, setting.TemplateVariableProduct, productName) + yaml = strings.ReplaceAll(yaml, setting.TemplateVariableService, serviceName) + return yaml +} + +func buildVariable(key string) string { + return fmt.Sprintf("{{.%s}}", key) +} + +// SyncServiceFromTemplate syncs services from (yaml|chart)template +func SyncServiceFromTemplate(userName, source, templateId, templateName string, logger *zap.SugaredLogger) error { + if source == setting.ServiceSourceTemplate { + return syncServicesFromYamlTemplate(userName, templateId, logger) + } else { + return syncServicesFromChartTemplate(userName, templateName, logger) + } +} + +func syncServicesFromYamlTemplate(userName, templateId string, logger *zap.SugaredLogger) error { + serviceList, err := commonrepo.NewServiceColl().ListMaxRevisionServicesByYamlTemplate(templateId) + if err != nil { + return err + } + servicesByProject := make(map[string][]*commonmodels.Service) + for _, service := range serviceList { + if !service.AutoSync { + continue + } + servicesByProject[service.ProductName] = append(servicesByProject[service.ProductName], service) + } + yamlTemplate, err := commonrepo.NewYamlTemplateColl().GetById(templateId) + if err != nil { + return fmt.Errorf("failed to find yaml template: %s, err: %s", templateId, err) + } + for _, services := range servicesByProject { + go func(pServices []*commonmodels.Service) { + for _, service := range pServices { + err := reloadServiceFromYamlTemplate(userName, service.ProductName, yamlTemplate, service) + if err != nil { + logger.Error(err) + title := fmt.Sprintf("从模板更新 [%s] 的 [%s] 服务失败", service.ProductName, service.ServiceName) + commonservice.SendErrorMessage(userName, title, "", err, logger) + } + } + }(services) + } + return nil +} + +func syncServicesFromChartTemplate(userName, templateName string, logger *zap.SugaredLogger) error { + chartTemplate, err := prepareChartTemplateData(templateName, log.SugaredLogger()) + if err != nil { + return err + } + + serviceList, err := commonrepo.NewServiceColl().ListMaxRevisionServicesByChartTemplate(templateName) + if err != nil { + return err + } + servicesByProject := make(map[string][]*commonmodels.Service) + for _, service := range serviceList { + if !service.AutoSync { + continue + } + servicesByProject[service.ProductName] = append(servicesByProject[service.ProductName], service) + } + + for _, services := range servicesByProject { + go func(pService []*commonmodels.Service) { + for _, service := range pService { + err := reloadServiceFromChartTemplate(service, chartTemplate) + if err != nil { + logger.Errorf("failed to reload service %s/%s from chart template, err: %s", service.ProductName, service.ServiceName, err) + title := fmt.Sprintf("从模板更新 [%s] 的 [%s] 服务失败", service.ProductName, service.ServiceName) + commonservice.SendErrorMessage(userName, title, "", err, logger) + } + } + }(services) + } + return nil +} + +func reloadServiceFromChartTemplate(service *commonmodels.Service, chartTemplate *ChartTemplateData) error { + variable, customYaml, err := buildChartTemplateVariables(service, chartTemplate.TemplateData) + if err != nil { + return err + } + + templateArgs := &CreateFromChartTemplate{ + TemplateName: chartTemplate.TemplateName, + ValuesYAML: customYaml, + Variables: variable, + } + args := &HelmServiceCreationArgs{ + HelmLoadSource: HelmLoadSource{}, + Name: service.ServiceName, + CreatedBy: "system", + ValuesData: nil, + CreationDetail: service.CreateFrom, + AutoSync: service.AutoSync, + } + ret, err := createOrUpdateHelmServiceFromChartTemplate(templateArgs, chartTemplate, service.ProductName, args, true, log.SugaredLogger()) + if err != nil { + return err + } + if len(ret.FailedServices) == 1 { + return errors.New(ret.FailedServices[0].Error) + } + return nil +} + +func buildYamlTemplateVariables(service *commonmodels.Service, template *commonmodels.YamlTemplate) ([]*Variable, error) { + variables := make([]*Variable, 0) + variableMap := make(map[string]*Variable) + for _, v := range template.Variables { + kv := &Variable{ + Key: v.Key, + Value: v.Value, + } + variableMap[v.Key] = kv + variables = append(variables, kv) + } + + creation := &commonmodels.CreateFromYamlTemplate{} + vbs := make([]*commonmodels.Variable, 0) + if service.CreateFrom != nil { + bs, err := json.Marshal(service.CreateFrom) + if err != nil { + log.Errorf("failed to marshal creation data: %s", err) + return variables, err + } + + err = json.Unmarshal(bs, creation) + if err != nil { + log.Errorf("failed to unmarshal creation data: %s", err) + return variables, err + } + for _, kv := range creation.Variables { + if tkv, ok := variableMap[kv.Key]; ok { + tkv.Value = kv.Value + } + } + } else { + creation.TemplateID = template.ID.Hex() + } + + for _, kv := range variables { + vbs = append(vbs, &commonmodels.Variable{ + Key: kv.Key, + Value: kv.Value, + }) + } + creation.Variables = vbs + service.CreateFrom = creation + + return variables, nil +} + +func buildChartTemplateVariables(service *commonmodels.Service, template *commonmodels.Chart) ([]*Variable, string, error) { + variables := make([]*Variable, 0) + variableMap := make(map[string]*Variable) + + for _, v := range template.ChartVariables { + kv := &Variable{ + Key: v.Key, + Value: v.Value, + } + variableMap[v.Key] = kv + variables = append(variables, kv) + } + + customYaml := "" + if service.CreateFrom != nil { + bs, err := json.Marshal(service.CreateFrom) + if err != nil { + log.Errorf("failed to marshal creation data: %s", err) + return variables, "", err + } + creation := &commonmodels.CreateFromChartTemplate{} + err = json.Unmarshal(bs, creation) + if err != nil { + log.Errorf("failed to unmarshal creation data: %s", err) + return variables, "", err + } + for _, kv := range creation.Variables { + if tkv, ok := variableMap[kv.Key]; ok { + tkv.Value = kv.Value + } + } + if creation.YamlData != nil { + customYaml = creation.YamlData.YamlContent + } + vbs := make([]*commonmodels.Variable, 0) + for _, kv := range variables { + vbs = append(vbs, &commonmodels.Variable{ + Key: kv.Key, + Value: kv.Value, + }) + } + creation.Variables = vbs + service.CreateFrom = creation + } + return variables, customYaml, nil +} + +func reloadServiceFromYamlTemplate(userName, projectName string, template *commonmodels.YamlTemplate, service *commonmodels.Service) error { + //extract variables from current service + variables, err := buildYamlTemplateVariables(service, template) + if err != nil { + return err + } + + renderedYaml := renderYamlFromTemplate(template.Content, projectName, service.ServiceName, variables) + svc := &commonmodels.Service{ + ServiceName: service.ServiceName, + Type: setting.K8SDeployType, + ProductName: projectName, + Source: setting.ServiceSourceTemplate, + Yaml: renderedYaml, + Visibility: setting.PrivateVisibility, + TemplateID: service.TemplateID, + CreateFrom: service.CreateFrom, + AutoSync: service.AutoSync, + } + _, err = CreateServiceTemplate(userName, svc, true, log.SugaredLogger()) + if err != nil { + return fmt.Errorf("failed to reload service template from template ID: %s, error : %s", service.TemplateID, err) + } + return nil +} diff --git a/pkg/microservice/aslan/core/service/service/types.go b/pkg/microservice/aslan/core/service/service/types.go index 0edc04a480ae5b70e302962aa21178b80baaefd4..027b29b884d914b837a0d45cdff1dd0150c374a5 100644 --- a/pkg/microservice/aslan/core/service/service/types.go +++ b/pkg/microservice/aslan/core/service/service/types.go @@ -27,9 +27,11 @@ import ( type LoadSource string const ( - LoadFromRepo LoadSource = "repo" + LoadFromRepo LoadSource = "repo" //exclude gerrit + LoadFromGerrit LoadSource = "gerrit" LoadFromPublicRepo LoadSource = "publicRepo" LoadFromChartTemplate LoadSource = "chartTemplate" + LoadFromChartRepo LoadSource = "chartRepo" ) type HelmLoadSource struct { @@ -38,21 +40,22 @@ type HelmLoadSource struct { type HelmServiceCreationArgs struct { HelmLoadSource - Name string `json:"name"` - CreatedBy string `json:"createdBy"` - CreateFrom interface{} `json:"createFrom"` -} - -type ValuesDataArgs struct { - YamlSource string `json:"yamlSource,omitempty"` - GitRepoConfig *service.RepoConfig `json:"gitRepoConfig,omitempty"` + Name string `json:"name"` + CreatedBy string `json:"createdBy"` + RequestID string `json:"-"` + AutoSync bool `json:"auto_sync"` + CreateFrom interface{} `json:"createFrom"` + ValuesData *service.ValuesDataArgs `json:"valuesData"` + CreationDetail interface{} `json:"-"` } type BulkHelmServiceCreationArgs struct { HelmLoadSource - CreateFrom interface{} `json:"createFrom"` - CreatedBy string `json:"createdBy"` - ValuesData *ValuesDataArgs `json:"valuesData"` + CreateFrom interface{} `json:"createFrom"` + CreatedBy string `json:"createdBy"` + RequestID string `json:"-"` + ValuesData *service.ValuesDataArgs `json:"valuesData"` + AutoSync bool `json:"auto_sync"` } type FailedService struct { @@ -68,6 +71,7 @@ type BulkHelmServiceCreationResponse struct { type CreateFromRepo struct { CodehostID int `json:"codehostID"` Owner string `json:"owner"` + Namespace string `json:"namespace"` Repo string `json:"repo"` Branch string `json:"branch"` Paths []string `json:"paths"` @@ -79,13 +83,15 @@ type CreateFromPublicRepo struct { } type CreateFromChartTemplate struct { - CodehostID int `json:"codehostID,omitempty"` - Owner string `json:"owner,omitempty"` - Repo string `json:"repo,omitempty"` - Branch string `json:"branch,omitempty"` - TemplateName string `json:"templateName"` - ValuesPaths []string `json:"valuesPaths,omitempty"` - ValuesYAML string `json:"valuesYAML,omitempty"` + TemplateName string `json:"templateName"` + ValuesYAML string `json:"valuesYAML"` + Variables []*Variable `json:"variables"` +} + +type CreateFromChartRepo struct { + ChartRepoName string `json:"chartRepoName"` + ChartName string `json:"chartName"` + ChartVersion string `json:"chartVersion"` } func PublicRepoToPrivateRepoArgs(args *CreateFromPublicRepo) (*CreateFromRepo, error) { @@ -117,6 +123,8 @@ func (a *HelmServiceCreationArgs) UnmarshalJSON(data []byte) error { a.CreateFrom = &CreateFromPublicRepo{} case LoadFromChartTemplate: a.CreateFrom = &CreateFromChartTemplate{} + case LoadFromChartRepo: + a.CreateFrom = &CreateFromChartRepo{} } type tmp HelmServiceCreationArgs diff --git a/pkg/microservice/aslan/core/setting/service/user.go b/pkg/microservice/aslan/core/setting/service/user.go deleted file mode 100644 index 45ef8f90c7353d3ca6657da0ece56be530ff94f2..0000000000000000000000000000000000000000 --- a/pkg/microservice/aslan/core/setting/service/user.go +++ /dev/null @@ -1,388 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package service - -import ( - "bytes" - "encoding/base64" - "errors" - "fmt" - "html/template" - "io/ioutil" - "os" - "path/filepath" - "strings" - "sync" - - "github.com/hashicorp/go-multierror" - "go.uber.org/zap" - corev1 "k8s.io/api/core/v1" - rbacv1beta1 "k8s.io/api/rbac/v1beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" - - "github.com/koderover/zadig/pkg/microservice/aslan/config" - commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" - commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" - "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/kube/wrapper" - "github.com/koderover/zadig/pkg/shared/poetry" - e "github.com/koderover/zadig/pkg/tool/errors" - krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client" - "github.com/koderover/zadig/pkg/tool/kube/getter" - "github.com/koderover/zadig/pkg/tool/kube/updater" - "github.com/koderover/zadig/pkg/types/permission" -) - -type kubeCfgTmplArgs struct { - CaCrtBase64 string - CaKeyBase64 string - KubeServerAddr string - Namespace string - User string - ClientCrtBase64 string - ClientKeyBase64 string -} - -func GetUserKubeConfig(userName string, userID int, superUser bool, log *zap.SugaredLogger) (string, error) { - username := strings.ToLower(userName) - username = config.NameSpaceRegex.ReplaceAllString(username, "-") - var ( - err error - productEnvs = make([]*commonmodels.Product, 0) - productNameMap map[string][]int64 - ) - if superUser { - productEnvs, err = commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{}) - if err != nil { - log.Errorf("GetUserKubeConfig Collection.Product.List error: %v", err) - return "", e.ErrListProducts.AddDesc(err.Error()) - } - - // 只管理同集群的资源，且排除状态为Terminating的namespace - productEnvs = filterProductWithoutExternalCluster(productEnvs) - } else { - //项目下所有公开环境 - publicProducts, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{IsPublic: true}) - if err != nil { - log.Errorf("GetUserKubeConfig Collection.Product.List List product error: %v", err) - return "", e.ErrListProducts.AddDesc(err.Error()) - } - // 只管理同集群的资源，且排除状态为Terminating的namespace - filterPublicProductEnvs := filterProductWithoutExternalCluster(publicProducts) - namespaceSet := sets.NewString() - for _, publicProduct := range filterPublicProductEnvs { - productEnvs = append(productEnvs, publicProduct) - namespaceSet.Insert(publicProduct.Namespace) - } - poetryClient := poetry.New(config.PoetryAPIServer(), config.PoetryAPIRootKey()) - productNameMap, err = poetryClient.GetUserProject(userID, log) - if err != nil { - log.Errorf("GetUserKubeConfig Collection.Product.List GetUserProject error: %v", err) - return "", e.ErrListProducts.AddDesc(err.Error()) - } - for productName, roleIDs := range productNameMap { - //用户关联角色所关联的环境 - for _, roleID := range roleIDs { - if roleID == setting.RoleOwnerID { - tmpProducts, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{Name: productName}) - if err != nil { - log.Errorf("GetUserKubeConfig Collection.Product.List product error: %v", err) - return "", e.ErrListProducts.AddDesc(err.Error()) - } - for _, product := range tmpProducts { - if !namespaceSet.Has(product.Namespace) { - productEnvs = append(productEnvs, product) - } - } - } else { - roleEnvs, err := poetryClient.ListRoleEnvs(productName, "", roleID, log) - if err != nil { - log.Errorf("GetUserKubeConfig Collection.Product.List ListRoleEnvs error: %v", err) - return "", e.ErrListProducts.AddDesc(err.Error()) - } - for _, roleEnv := range roleEnvs { - product, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{Name: productName, EnvName: roleEnv.EnvName}) - if err != nil { - log.Errorf("GetUserKubeConfig Collection.Product.List Find product error: %v", err) - return "", e.ErrListProducts.AddDesc(err.Error()) - } - productEnvs = append(productEnvs, product) - } - } - } - } - } - saNamespace := config.Namespace() - if err := ensureServiceAccount(saNamespace, username, log); err != nil { - return "", err - } - - var ( - wg sync.WaitGroup - pool = make(chan int, 20) - errList = new(multierror.Error) - ) - for _, productEnv := range productEnvs { - namespace := productEnv.Namespace - productName := productEnv.ProductName - - if _, found, err := getter.GetNamespace(namespace, krkubeclient.Client()); err != nil || !found { - log.Error(err) - continue - } - - wg.Add(1) - pool <- 1 - go func() { - defer func() { - wg.Done() - <-pool - }() - if err := ensureUserRole(namespace, username, productName, userID, superUser, log); err != nil { - log.Error(err) - errList = multierror.Append(errList, err) - } - - if err := ensureUserRoleBinding(saNamespace, namespace, username); err != nil { - log.Error(err) - errList = multierror.Append(errList, err) - } - }() - } - wg.Wait() - close(pool) - if err := errList.ErrorOrNil(); err != nil { - log.Error(err) - return "", err - } - - if err := createK8sSSLCert(saNamespace, username); err != nil { - log.Errorf("[%s] createSSLCert error: %v", username, err) - return "", err - } - - caCrtBase64, err := ioutil.ReadFile(filepath.Join(os.TempDir(), username, "ca.crt")) - if err != nil { - return "", fmt.Errorf("get client.crt error: %v", err) - } - - caKeyBase64, err := ioutil.ReadFile(filepath.Join(os.TempDir(), username, "ca.key")) - if err != nil { - return "", fmt.Errorf("get client.key error: %v", err) - } - - args := &kubeCfgTmplArgs{ - KubeServerAddr: config.KubeServerAddr(), - User: username, - CaCrtBase64: string(caCrtBase64), - CaKeyBase64: string(caKeyBase64), - } - - return renderCfgTmpl(args) -} - -func filterProductWithoutExternalCluster(products []*commonmodels.Product) []*commonmodels.Product { - var ret []*commonmodels.Product - kubeClient := krkubeclient.Client() - for _, product := range products { - // 过滤跨集群 - if product.ClusterID != "" { - continue - } - // 过滤外部环境托管 - sources := sets.NewString(setting.SourceFromZadig, setting.HelmDeployType) - // Compatible with the environment source created in the onboarding process is empty - if product.Source != "" && !sources.Has(product.Source) { - continue - } - // 过滤状态为Terminating的namespace - namespace, found, err := getter.GetNamespace(product.Namespace, kubeClient) - if !found || err != nil || wrapper.Namespace(namespace).Terminating() { - continue - } - - ret = append(ret, product) - } - - return ret -} - -func ensureServiceAccount(namespace, username string, log *zap.SugaredLogger) error { - serviceAccount := &corev1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{ - Name: username + "-sa", - Namespace: namespace, - }, - } - if err := updater.CreateServiceAccount(serviceAccount, krkubeclient.Client()); err != nil { - log.Errorf("CreateServiceAccount err: %+v", err) - return nil - } - return nil -} - -func ensureUserRole(namespace, username, productName string, userID int, superUser bool, log *zap.SugaredLogger) error { - poetryClient := poetry.New(config.PoetryAPIServer(), config.PoetryAPIRootKey()) - roleName := fmt.Sprintf("%s-role", username) - verbs := []string{"get", "list", "watch"} - if poetryClient.HasOperatePermission(productName, permission.TestEnvManageUUID, userID, superUser, log) { - verbs = []string{"*"} - } - role := &rbacv1beta1.Role{ - ObjectMeta: metav1.ObjectMeta{ - Name: roleName, - Namespace: namespace, - }, - Rules: []rbacv1beta1.PolicyRule{ - rbacv1beta1.PolicyRule{ - APIGroups: []string{"*"}, - Resources: []string{"*"}, - Verbs: verbs, - }, - rbacv1beta1.PolicyRule{ - APIGroups: []string{"*"}, - Resources: []string{ - "limitranges", - "resourcequotas", - }, - Verbs: []string{"get", "list", "watch"}, - }, - }, - } - - old, found, err := getter.GetRole(namespace, roleName, krkubeclient.Client()) - if err == nil && found { - old.Rules = role.Rules - if err := updater.UpdateRole(old, krkubeclient.Client()); err != nil { - return err - } - } else { - if err := updater.CreateRole(role, krkubeclient.Client()); err != nil { - return err - } - } - - return nil -} - -func ensureUserRoleBinding(saNamespace, namespace, username string) error { - roleName := fmt.Sprintf("%s-role", username) - roleBindName := fmt.Sprintf("%s-role-bind", username) - rolebinding := &rbacv1beta1.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: roleBindName, - Namespace: namespace, - }, - RoleRef: rbacv1beta1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "Role", - Name: roleName, - }, - Subjects: []rbacv1beta1.Subject{ - rbacv1beta1.Subject{ - //APIGroup: "rbac.authorization.k8s.io", - //Kind: "User", - //Name: namespace, - Kind: "ServiceAccount", - Name: username + "-sa", - Namespace: saNamespace, - }, - }, - } - _, found, err := getter.GetRoleBinding(namespace, roleBindName, krkubeclient.Client()) - if err != nil || !found { - if err := updater.CreateRoleBinding(rolebinding, krkubeclient.Client()); err != nil { - return err - } - } else { - if err := updater.UpdateRoleBinding(rolebinding, krkubeclient.Client()); err != nil { - return err - } - } - return nil -} - -func createK8sSSLCert(namespace, username string) error { - workDir := filepath.Join(os.TempDir(), username) - if err := os.MkdirAll(workDir, os.ModePerm); err != nil { - return err - } - - kubeClient := krkubeclient.Client() - - sa, found, err := getter.GetServiceAccount(namespace, username+"-sa", kubeClient) - if err != nil { - return err - } else if !found { - return errors.New("sa not fonud") - } else if len(sa.Secrets) == 0 { - return errors.New("no secrets in sa") - } - secret, found, err := getter.GetSecret(namespace, sa.Secrets[0].Name, kubeClient) - if err != nil { - return err - } else if !found { - return errors.New("secret not found") - } - keyPath := filepath.Join(workDir, "ca.key") - certPath := filepath.Join(workDir, "ca.crt") - - err = ioutil.WriteFile(keyPath, secret.Data["token"], 0644) - if err != nil { - return err - } - cert := make([]byte, base64.StdEncoding.EncodedLen(len(secret.Data["ca.crt"]))) - base64.StdEncoding.Encode(cert, secret.Data["ca.crt"]) - err = ioutil.WriteFile(certPath, cert, 0644) - if err != nil { - return err - } - return nil -} - -func renderCfgTmpl(args *kubeCfgTmplArgs) (string, error) { - buf := new(bytes.Buffer) - t := template.Must(template.New("cfg").Parse(kubeCfgTmpl)) - err := t.Execute(buf, args) - if err != nil { - return "", err - } - return buf.String(), nil -} - -const kubeCfgTmpl = ` -apiVersion: v1 -clusters: -- cluster: - certificate-authority-data: {{.CaCrtBase64}} - server: {{.KubeServerAddr}} - name: koderover -contexts: -- context: - cluster: koderover - user: {{.User}} - name: {{.User}}@kubernetes -current-context: {{.User}}@kubernetes -kind: Config -preferences: {} -users: -- name: {{.User}} - user: - token: {{.CaKeyBase64}} - client-key-data: {{.CaCrtBase64}} -` diff --git a/pkg/microservice/aslan/core/stat/handler/build.go b/pkg/microservice/aslan/core/stat/handler/build.go new file mode 100644 index 0000000000000000000000000000000000000000..f9dba4c3380f98c9ded2f6ba1a6292a1c6cd9ed6 --- /dev/null +++ b/pkg/microservice/aslan/core/stat/handler/build.go @@ -0,0 +1,47 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/stat/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/stat/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +type GetBuildStatArgs struct { + StartDate int64 `json:"startDate" form:"startDate,default:0"` + EndDate int64 `json:"endDate" form:"endDate,default:0"` +} + +func GetBuildStat(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(GetBuildStatArgs) + if err := c.ShouldBindQuery(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } + + ctx.Resp, ctx.Err = service.GetBuildTotalAndSuccess(&models.BuildStatOption{ + StartDate: args.StartDate, + EndDate: args.EndDate, + }, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/stat/handler/build_stat.go b/pkg/microservice/aslan/core/stat/handler/build_stat.go new file mode 100644 index 0000000000000000000000000000000000000000..d70235f582d77a801183bc5e25d6b983a4c0a728 --- /dev/null +++ b/pkg/microservice/aslan/core/stat/handler/build_stat.go @@ -0,0 +1,119 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/stat/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +type getStatReq struct { + StartDate int64 `json:"startDate,omitempty"` + EndDate int64 `json:"endDate,omitempty"` + ProductNames []string `json:"productNames"` +} + +func GetAllPipelineTask(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Err = service.GetAllPipelineTask(ctx.Logger) +} + +func GetBuildDailyAverageMeasure(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + //params validate + + args := new(getStatReq) + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + + ctx.Resp, ctx.Err = service.GetBuildDailyAverageMeasure(args.StartDate, args.EndDate, args.ProductNames, ctx.Logger) +} + +func GetBuildDailyMeasure(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + //params validate + args := new(getStatReq) + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + + ctx.Resp, ctx.Err = service.GetBuildDailyMeasure(args.StartDate, args.EndDate, args.ProductNames, ctx.Logger) +} + +func GetBuildHealthMeasure(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + //params validate + args := new(getStatReq) + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + + ctx.Resp, ctx.Err = service.GetBuildHealthMeasure(args.StartDate, args.EndDate, args.ProductNames, ctx.Logger) +} + +func GetLatestTenBuildMeasure(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + //params validate + // TODO: ????????????????? + args := new(getStatReq) + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + + // TODO: ↓ only this is used + ctx.Resp, ctx.Err = service.GetLatestTenBuildMeasure(args.ProductNames, ctx.Logger) +} + +func GetTenDurationMeasure(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + //params validate + args := new(getStatReq) + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + + ctx.Resp, ctx.Err = service.GetTenDurationMeasure(args.StartDate, args.EndDate, args.ProductNames, ctx.Logger) +} + +func GetBuildTrendMeasure(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + //params validate + args := new(getStatReq) + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + + ctx.Resp, ctx.Err = service.GetBuildTrendMeasure(args.StartDate, args.EndDate, args.ProductNames, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/stat/handler/deploy.go b/pkg/microservice/aslan/core/stat/handler/deploy.go new file mode 100644 index 0000000000000000000000000000000000000000..51f3832f06b5257945fe13129a8f688c805617be --- /dev/null +++ b/pkg/microservice/aslan/core/stat/handler/deploy.go @@ -0,0 +1,47 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/stat/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/stat/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +type GetDeployStatArgs struct { + StartDate int64 `json:"startDate" form:"startDate,default:0"` + EndDate int64 `json:"endDate" form:"endDate,default:0"` +} + +func GetDeployStat(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(GetDeployStatArgs) + if err := c.ShouldBindQuery(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } + + ctx.Resp, ctx.Err = service.GetDeployDailyTotalAndSuccess(&models.DeployStatOption{ + StartDate: args.StartDate, + EndDate: args.EndDate, + }, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/stat/handler/deploy_stat.go b/pkg/microservice/aslan/core/stat/handler/deploy_stat.go new file mode 100644 index 0000000000000000000000000000000000000000..24ca1c1ececd6b6d3fa5767baca7085eb785ed38 --- /dev/null +++ b/pkg/microservice/aslan/core/stat/handler/deploy_stat.go @@ -0,0 +1,92 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/stat/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +func InitDeployStat(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Err = service.InitDeployStat(ctx.Logger) +} + +func GetPipelineHealthMeasure(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + //params validate + args := new(getStatReq) + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + ctx.Resp, ctx.Err = service.GetPipelineHealthMeasure(args.StartDate, args.EndDate, args.ProductNames, ctx.Logger) +} + +func GetDeployHealthMeasure(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + //params validate + args := new(getStatReq) + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + ctx.Resp, ctx.Err = service.GetDeployHealthMeasure(args.StartDate, args.EndDate, args.ProductNames, ctx.Logger) +} + +func GetDeployWeeklyMeasure(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + //params validate + args := new(getStatReq) + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + ctx.Resp, ctx.Err = service.GetDeployWeeklyMeasure(args.StartDate, args.EndDate, args.ProductNames, ctx.Logger) +} + +func GetDeployTopFiveHigherMeasure(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + //params validate + args := new(getStatReq) + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + ctx.Resp, ctx.Err = service.GetDeployTopFiveHigherMeasure(args.StartDate, args.EndDate, args.ProductNames, ctx.Logger) +} + +func GetDeployTopFiveFailureMeasure(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + //params validate + args := new(getStatReq) + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + ctx.Resp, ctx.Err = service.GetDeployTopFiveFailureMeasure(args.StartDate, args.EndDate, args.ProductNames, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/stat/handler/overview.go b/pkg/microservice/aslan/core/stat/handler/overview.go new file mode 100644 index 0000000000000000000000000000000000000000..74646eaf3509c0da334a49c124d2b6a5fcd6a91c --- /dev/null +++ b/pkg/microservice/aslan/core/stat/handler/overview.go @@ -0,0 +1,31 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/stat/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" +) + +func GetOverviewStat(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.GetOverviewStat(ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/stat/handler/router.go b/pkg/microservice/aslan/core/stat/handler/router.go new file mode 100644 index 0000000000000000000000000000000000000000..fa409dbbd7c943a3f6041ee5ab82d103ce74fafd --- /dev/null +++ b/pkg/microservice/aslan/core/stat/handler/router.go @@ -0,0 +1,59 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" +) + +type Router struct{} + +func (*Router) Inject(router *gin.RouterGroup) { + dashboard := router.Group("dashboard") + { + dashboard.GET("/overview", GetOverviewStat) + dashboard.GET("/build", GetBuildStat) + dashboard.GET("/deploy", GetDeployStat) + dashboard.GET("/test", GetTestDashboard) + } + + quality := router.Group("quality") + { + //buildStat + quality.POST("/initBuildStat", GetAllPipelineTask) + quality.POST("/buildAverageMeasure", GetBuildDailyAverageMeasure) + quality.POST("/buildDailyMeasure", GetBuildDailyMeasure) + quality.POST("/buildHealthMeasure", GetBuildHealthMeasure) + quality.POST("/buildLatestTenMeasure", GetLatestTenBuildMeasure) + quality.POST("/buildTenDurationMeasure", GetTenDurationMeasure) + quality.POST("/buildTrend", GetBuildTrendMeasure) + //testStat + quality.POST("/initTestStat", InitTestStat) + quality.POST("/testAverageMeasure", GetTestAverageMeasure) + quality.POST("/testCaseMeasure", GetTestCaseMeasure) + quality.POST("/testDeliveryDeploy", GetTestDeliveryDeployMeasure) + quality.POST("/testHealthMeasure", GetTestHealthMeasure) + quality.POST("/testTrend", GetTestTrendMeasure) + //deployStat + quality.POST("/initDeployStat", InitDeployStat) + quality.POST("/pipelineHealthMeasure", GetPipelineHealthMeasure) + quality.POST("/deployHealthMeasure", GetDeployHealthMeasure) + quality.POST("/deployWeeklyMeasure", GetDeployWeeklyMeasure) + quality.POST("/deployTopFiveHigherMeasure", GetDeployTopFiveHigherMeasure) + quality.POST("/deployTopFiveFailureMeasure", GetDeployTopFiveFailureMeasure) + } +} diff --git a/pkg/microservice/aslan/core/stat/handler/test.go b/pkg/microservice/aslan/core/stat/handler/test.go new file mode 100644 index 0000000000000000000000000000000000000000..8c6899e008127eed32b92d752889aff87bb821e0 --- /dev/null +++ b/pkg/microservice/aslan/core/stat/handler/test.go @@ -0,0 +1,31 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/stat/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" +) + +func GetTestDashboard(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.GetTestDashboard(ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/stat/handler/test_stat.go b/pkg/microservice/aslan/core/stat/handler/test_stat.go new file mode 100644 index 0000000000000000000000000000000000000000..49417ccbbdf8c1c9c9b53feb23f4db177d54e36a --- /dev/null +++ b/pkg/microservice/aslan/core/stat/handler/test_stat.go @@ -0,0 +1,92 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/stat/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +func InitTestStat(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Err = service.InitTestStat(ctx.Logger) +} + +func GetTestAverageMeasure(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + //params validate + args := new(getStatReq) + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + ctx.Resp, ctx.Err = service.GetTestAverageMeasure(args.StartDate, args.EndDate, args.ProductNames, ctx.Logger) +} + +func GetTestCaseMeasure(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + //params validate + args := new(getStatReq) + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + ctx.Resp, ctx.Err = service.GetTestCaseMeasure(args.StartDate, args.EndDate, args.ProductNames, ctx.Logger) +} + +func GetTestDeliveryDeployMeasure(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + //params validate + args := new(getStatReq) + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + ctx.Resp, ctx.Err = service.GetTestDeliveryDeployMeasure(args.StartDate, args.EndDate, args.ProductNames, ctx.Logger) +} + +func GetTestHealthMeasure(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + //params validate + args := new(getStatReq) + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + ctx.Resp, ctx.Err = service.GetTestHealthMeasure(args.StartDate, args.EndDate, args.ProductNames, ctx.Logger) +} + +func GetTestTrendMeasure(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + //params validate + args := new(getStatReq) + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + ctx.Resp, ctx.Err = service.GetTestTrendMeasure(args.StartDate, args.EndDate, args.ProductNames, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/stat/repository/models/build.go b/pkg/microservice/aslan/core/stat/repository/models/build.go new file mode 100644 index 0000000000000000000000000000000000000000..67b30b125f2df5c9399ad74e37e56e45c8ec35f7 --- /dev/null +++ b/pkg/microservice/aslan/core/stat/repository/models/build.go @@ -0,0 +1,52 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +type BuildStat struct { + ProductName string `bson:"product_name" json:"productName"` + TotalSuccess int `bson:"total_success" json:"totalSuccess"` + TotalFailure int `bson:"total_failure" json:"totalFailure"` + TotalTimeout int `bson:"total_timeout" json:"totalTimeout"` + TotalDuration int64 `bson:"total_duration" json:"totalDuration"` + TotalBuildCount int `bson:"total_build_count" json:"totalBuildCount"` + MaxDuration int64 `bson:"max_duration" json:"maxDuration"` + MaxDurationPipeline *PipelineInfo `bson:"max_duration_pipeline" json:"maxDurationPipeline"` + Date string `bson:"date" json:"date"` + CreateTime int64 `bson:"create_time" json:"createTime"` + UpdateTime int64 `bson:"update_time" json:"updateTime"` +} + +// PipelineInfo +type PipelineInfo struct { + TaskID int64 `bson:"task_id" json:"taskId"` + PipelineName string `bson:"pipeline_name" json:"pipelineName"` + Type string `bson:"type" json:"type"` + MaxDuration int64 `bson:"max_duration" json:"maxDuration"` +} + +type BuildStatOption struct { + StartDate int64 + EndDate int64 + Limit int + Skip int + IsAsc bool + ProductNames []string +} + +func (BuildStat) TableName() string { + return "build_stat" +} diff --git a/pkg/microservice/aslan/core/stat/repository/models/deploy.go b/pkg/microservice/aslan/core/stat/repository/models/deploy.go new file mode 100644 index 0000000000000000000000000000000000000000..2baaaacb7cdd23398a86b15c6ab942c5dfba55f6 --- /dev/null +++ b/pkg/microservice/aslan/core/stat/repository/models/deploy.go @@ -0,0 +1,47 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +type DeployStatOption struct { + StartDate int64 + EndDate int64 + Limit int + Skip int + IsAsc bool + IsMaxDeploy bool + ProductNames []string +} + +type DeployStat struct { + ProductName string `bson:"product_name" json:"productName"` + TotalTaskSuccess int `bson:"total_task_success" json:"totalTaskSuccess"` + TotalTaskFailure int `bson:"total_task_failure" json:"totalTaskFailure"` + TotalDeploySuccess int `bson:"total_deploy_success" json:"totalDeploySuccess"` + TotalDeployFailure int `bson:"total_deploy_failure" json:"totalDeployFailure"` + MaxDeployServiceNum int `bson:"max_deploy_service_num" json:"maxDeployServiceNum"` + MaxDeployServiceFailureNum int `bson:"max_deploy_service_failure_num" json:"maxDeployServiceFailureNum"` + MaxDeployServiceName string `bson:"max_deploy_service_name" json:"maxDeployServiceName"` + MaxDeployFailureServiceNum int `bson:"max_deploy_failure_service_num" json:"maxDeployFailureServiceNum"` + MaxDeployFailureServiceName string `bson:"max_deploy_failure_service_name" json:"maxDeployFailureServiceName"` + Date string `bson:"date" json:"date"` + CreateTime int64 `bson:"create_time" json:"createTime"` + UpdateTime int64 `bson:"update_time" json:"updateTime"` +} + +func (DeployStat) TableName() string { + return "deploy_stat" +} diff --git a/pkg/microservice/aslan/core/stat/repository/models/test.go b/pkg/microservice/aslan/core/stat/repository/models/test.go new file mode 100644 index 0000000000000000000000000000000000000000..d2addd021c7a4e336f03aebdd3c2dc87ef80cc89 --- /dev/null +++ b/pkg/microservice/aslan/core/stat/repository/models/test.go @@ -0,0 +1,35 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +type TestStat struct { + ProductName string `bson:"product_name" json:"productName"` + TotalSuccess int `bson:"total_success" json:"totalSuccess"` + TotalFailure int `bson:"total_failure" json:"totalFailure"` + TotalTimeout int `bson:"total_timeout" json:"totalTimeout"` + TotalDuration int64 `bson:"total_duration" json:"totalDuration"` + TotalTestCount int `bson:"total_test_count" json:"totalTestCount"` + TotalDeployCount int `bson:"total_deploy_count" json:"totalDeployCount"` + TotalTestCase int `bson:"total_test_case" json:"totalTestCase"` + Date string `bson:"date" json:"date"` + CreateTime int64 `bson:"create_time" json:"createTime"` + UpdateTime int64 `bson:"update_time" json:"updateTime"` +} + +func (TestStat) TableName() string { + return "test_stat" +} diff --git a/pkg/microservice/aslan/core/stat/repository/mongodb/build.go b/pkg/microservice/aslan/core/stat/repository/mongodb/build.go new file mode 100644 index 0000000000000000000000000000000000000000..7d32bf819b53cffbbc30262772536c1b2a03d15b --- /dev/null +++ b/pkg/microservice/aslan/core/stat/repository/mongodb/build.go @@ -0,0 +1,243 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "errors" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + models "github.com/koderover/zadig/pkg/microservice/aslan/core/stat/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type BuildPipeResp struct { + ID BuildItem `bson:"_id" json:"_id"` + TotalSuccess int `bson:"total_success" json:"total_success"` + TotalBuildCount int `bson:"total_build_count" json:"total_build_count"` +} + +type BuildItem struct { + Date string `bson:"date" json:"date"` + TotalSuccess int `bson:"total_success" json:"total_success"` + TotalBuildCount int `bson:"total_build_count" json:"total_build_count"` +} + +type BuildDailyPipeResp struct { + ID BuildDailyItem `bson:"_id" json:"_id"` + TotalSuccess int `bson:"total_success" json:"total_success"` + TotalFailure int `bson:"total_failure" json:"total_failure"` + TotalBuildCount int `bson:"total_build_count" json:"total_build_count"` +} + +type BuildDailyItem struct { + Date string `bson:"date" json:"date"` + TotalSuccess int `bson:"total_success" json:"total_success"` + TotalFailure int `bson:"total_failure" json:"total_failure"` + TotalBuildCount int `bson:"total_build_count" json:"total_build_count"` +} + +type BuildStatColl struct { + *mongo.Collection + + coll string +} + +func NewBuildStatColl() *BuildStatColl { + name := models.BuildStat{}.TableName() + return &BuildStatColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} +} + +func (c *BuildStatColl) GetCollectionName() string { + return c.coll +} + +func (c *BuildStatColl) EnsureIndex(ctx context.Context) error { + mod := mongo.IndexModel{ + Keys: bson.D{ + bson.E{Key: "product_name", Value: 1}, + bson.E{Key: "date", Value: 1}, + }, + Options: options.Index().SetUnique(true), + } + + _, err := c.Indexes().CreateOne(ctx, mod) + + return err +} + +func (c *BuildStatColl) Create(args *models.BuildStat) error { + if args == nil { + return errors.New("nil buildStat args") + } + + _, err := c.InsertOne(context.TODO(), args) + return err +} + +func (c *BuildStatColl) Update(args *models.BuildStat) error { + if args == nil { + return errors.New("nil buildStat args") + } + + query := bson.M{"date": args.Date, "product_name": args.ProductName} + update := bson.M{"$set": args} + _, err := c.UpdateOne(context.TODO(), query, update) + return err +} + +func (c *BuildStatColl) FindCount() (int, error) { + count, err := c.EstimatedDocumentCount(context.TODO()) + if err != nil { + return 0, err + } + return int(count), nil +} + +func (c *BuildStatColl) ListBuildStat(option *models.BuildStatOption) ([]*models.BuildStat, error) { + resp := make([]*models.BuildStat, 0) + query := bson.M{} + + if len(option.ProductNames) > 0 { + query["product_name"] = bson.M{"$in": option.ProductNames} + } + + if option.StartDate > 0 { + query["create_time"] = bson.M{"$gte": option.StartDate, "$lte": option.EndDate} + } + + opt := &options.FindOptions{} + + if option.Limit > 0 { + opt.SetSort(bson.D{{"max_duration", -1}}).SetSkip(int64(option.Skip)).SetLimit(int64(option.Limit)) + } else if option.IsAsc { + opt.SetSort(bson.D{{"create_time", 1}}) + } else { + opt.SetSort(bson.D{{"create_time", -1}}) + } + + cursor, err := c.Collection.Find(context.TODO(), query, opt) + if err != nil { + return nil, err + } + + err = cursor.All(context.TODO(), &resp) + if err != nil { + return nil, err + } + return resp, nil +} + +func (c *BuildStatColl) GetBuildTotalAndSuccess() ([]*BuildItem, error) { + var result []*BuildPipeResp + var pipeline []bson.M + var resp []*BuildItem + + pipeline = append(pipeline, + bson.M{ + "$group": bson.M{ + "_id": bson.M{ + "date": "$date", + }, + "total_success": bson.M{ + "$sum": "$total_success", + }, + "total_build_count": bson.M{ + "$sum": "$total_build_count", + }, + }, + }) + + cursor, err := c.Aggregate(context.TODO(), pipeline) + if err != nil { + return nil, err + } + if err := cursor.All(context.TODO(), &result); err != nil { + return nil, err + } + for _, res := range result { + buildItem := &BuildItem{ + TotalSuccess: res.TotalSuccess, + TotalBuildCount: res.TotalBuildCount, + } + resp = append(resp, buildItem) + } + + return resp, nil +} + +func (c *BuildStatColl) GetBuildDailyTotal(args *models.BuildStatOption) ([]*BuildDailyItem, error) { + var result []*BuildDailyPipeResp + var resp []*BuildDailyItem + + pipeline := []bson.M{{ + "$sort": bson.M{ + "create_time": 1, + }, + }} + + if args.StartDate > 0 || args.EndDate > 0 { + timeRange := bson.M{} + if args.StartDate > 0 { + timeRange["$gte"] = args.StartDate + } + if args.EndDate > 0 { + timeRange["$lte"] = args.EndDate + } + pipeline = append(pipeline, bson.M{"$match": bson.M{"create_time": timeRange}}) + } + + pipeline = append(pipeline, bson.M{ + "$group": bson.M{ + "_id": bson.M{ + "date": "$date", + }, + "total_success": bson.M{ + "$sum": "$total_success", + }, + "total_failure": bson.M{ + "$sum": "$total_failure", + }, + "total_build_count": bson.M{ + "$sum": "$total_build_count", + }, + }, + }) + + cursor, err := c.Aggregate(context.TODO(), pipeline) + if err != nil { + return nil, err + } + if err := cursor.All(context.TODO(), &result); err != nil { + return nil, err + } + for _, res := range result { + buildDailyItem := &BuildDailyItem{ + Date: res.ID.Date, + TotalSuccess: res.TotalSuccess, + TotalFailure: res.TotalFailure, + TotalBuildCount: res.TotalBuildCount, + } + resp = append(resp, buildDailyItem) + } + + return resp, nil +} diff --git a/pkg/microservice/aslan/core/stat/repository/mongodb/deploy.go b/pkg/microservice/aslan/core/stat/repository/mongodb/deploy.go new file mode 100644 index 0000000000000000000000000000000000000000..24b44f5a3a1dc587e782459b53843f7bb50d920d --- /dev/null +++ b/pkg/microservice/aslan/core/stat/repository/mongodb/deploy.go @@ -0,0 +1,338 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "errors" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + models "github.com/koderover/zadig/pkg/microservice/aslan/core/stat/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type DeployTotalPipeResp struct { + ID DeployTotalItem `bson:"_id" json:"_id"` + TotalSuccess int `bson:"total_deploy_success" json:"total_deploy_success"` + TotalFailure int `bson:"total_deploy_failure" json:"total_deploy_failure"` +} + +type DeployTotalItem struct { + ProductName string `bson:"product_name" json:"product_name"` + TotalSuccess int `bson:"total_deploy_success" json:"total_deploy_success"` + TotalFailure int `bson:"total_deploy_failure" json:"total_deploy_failure"` +} + +type DeployDailyPipeResp struct { + ID DeployDailyItem `bson:"_id" json:"_id"` + TotalSuccess int `bson:"total_deploy_success" json:"total_deploy_success"` + TotalFailure int `bson:"total_deploy_failure" json:"total_deploy_failure"` +} + +type DeployDailyItem struct { + Date string `bson:"date" json:"date"` + TotalSuccess int `bson:"total_deploy_success" json:"total_deploy_success"` + TotalFailure int `bson:"total_deploy_failure" json:"total_deploy_failure"` +} + +type DeployPipeResp struct { + ID DeployPipeInfo `bson:"_id" json:"_id"` + MaxDeployServiceNum int `bson:"max_deploy_service_num" json:"maxDeployServiceNum"` +} + +type DeployPipeInfo struct { + MaxDeployServiceName string `bson:"max_deploy_service_name" json:"maxDeployServiceName"` +} + +type DeployFailurePipeResp struct { + ID DeployFailurePipeInfo `bson:"_id" json:"_id"` + MaxDeployFailureServiceNum int `bson:"max_deploy_failure_service_num" json:"maxDeployFailureServiceNum"` +} + +type DeployFailurePipeInfo struct { + ProductName string `bson:"product_name" json:"productName"` + MaxDeployFailureServiceName string `bson:"max_deploy_failure_service_name" json:"maxDeployFailureServiceName"` +} + +type DeployStatColl struct { + *mongo.Collection + + coll string +} + +func NewDeployStatColl() *DeployStatColl { + name := models.DeployStat{}.TableName() + return &DeployStatColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} +} + +func (c *DeployStatColl) GetCollectionName() string { + return c.coll +} + +func (c *DeployStatColl) EnsureIndex(ctx context.Context) error { + mod := mongo.IndexModel{ + Keys: bson.D{ + bson.E{Key: "product_name", Value: 1}, + bson.E{Key: "date", Value: 1}, + }, + Options: options.Index().SetUnique(true), + } + + _, err := c.Indexes().CreateOne(ctx, mod) + + return err +} + +func (c *DeployStatColl) Create(args *models.DeployStat) error { + if args == nil { + return errors.New("nil deployStat args") + } + + _, err := c.InsertOne(context.TODO(), args) + return err +} + +type DeployStatGetOption struct { + MaxDeployServiceNum int + ServiceName string +} + +func (c *DeployStatColl) Get(args *DeployStatGetOption) (*models.DeployStat, error) { + ret := new(models.DeployStat) + query := bson.M{"max_deploy_service_num": args.MaxDeployServiceNum, "max_deploy_service_name": args.ServiceName} + err := c.FindOne(context.TODO(), query).Decode(ret) + if err != nil { + return nil, err + } + return ret, nil +} + +func (c *DeployStatColl) Update(args *models.DeployStat) error { + if args == nil { + return errors.New("nil deployStat args") + } + + query := bson.M{ + "date": args.Date, + "product_name": args.ProductName, + } + update := bson.M{"$set": args} + _, err := c.UpdateOne(context.TODO(), query, update) + return err +} + +func (c *DeployStatColl) FindCount() (int, error) { + count, err := c.CountDocuments(context.TODO(), bson.M{}) + return int(count), err +} + +func (c *DeployStatColl) GetDeployTotalAndSuccess() ([]*DeployTotalItem, error) { + var result []*DeployTotalPipeResp + var resp []*DeployTotalItem + + pipeline := []bson.M{{ + "$group": bson.M{ + "_id": bson.M{ + "product_name": "$product_name", + }, + "total_deploy_success": bson.M{ + "$sum": "$total_deploy_success", + }, + "total_deploy_failure": bson.M{ + "$sum": "$total_deploy_failure", + }, + }, + }} + + cursor, err := c.Aggregate(context.TODO(), pipeline) + if err != nil { + return nil, err + } + if err := cursor.All(context.TODO(), &result); err != nil { + return nil, err + } + for _, res := range result { + deployItem := &DeployTotalItem{ + ProductName: res.ID.ProductName, + TotalSuccess: res.TotalSuccess, + TotalFailure: res.TotalFailure, + } + resp = append(resp, deployItem) + } + + return resp, nil +} + +func (c *DeployStatColl) ListDeployStat(option *models.DeployStatOption) ([]*models.DeployStat, error) { + ret := make([]*models.DeployStat, 0) + query := bson.M{} + if option.Limit > 0 { + var deployStatQuery []bson.M + if len(option.ProductNames) > 0 { + query["product_name"] = bson.M{"$in": option.ProductNames} + } + if option.StartDate > 0 { + query["create_time"] = bson.M{ + "$gte": option.StartDate, + "$lte": option.EndDate, + } + } + deployStatQuery = append(deployStatQuery, bson.M{"$match": query}) + + if option.IsMaxDeploy { + var results []DeployPipeResp + deployStatQuery = append(deployStatQuery, + bson.M{"$group": bson.M{ + "_id": bson.M{"max_deploy_service_name": "$max_deploy_service_name"}, + "max_deploy_service_num": bson.M{"$max": "$max_deploy_service_num"}}, + }) + deployStatQuery = append(deployStatQuery, bson.M{"$sort": bson.M{"max_deploy_service_num": -1}}) + deployStatQuery = append(deployStatQuery, bson.M{"$limit": 5}) + deployStatQuery = append(deployStatQuery, bson.M{"$skip": 0}) + + cursor, err := c.Aggregate(context.TODO(), deployStatQuery) + if err != nil { + return nil, err + } + if err := cursor.All(context.TODO(), &results); err != nil { + return nil, err + } + for _, result := range results { + deployItem := &models.DeployStat{ + MaxDeployServiceName: result.ID.MaxDeployServiceName, + MaxDeployServiceNum: result.MaxDeployServiceNum, + } + ret = append(ret, deployItem) + } + } else { + var results []DeployFailurePipeResp + + deployStatQuery = append(deployStatQuery, + bson.M{"$group": bson.M{ + "_id": bson.M{ + "max_deploy_failure_service_name": "$max_deploy_failure_service_name", + "product_name": "$product_name", + }, + "max_deploy_failure_service_num": bson.M{ + "$max": "$max_deploy_failure_service_num", + }, + }}) + deployStatQuery = append(deployStatQuery, bson.M{"$sort": bson.M{"max_deploy_failure_service_num": -1}}) + deployStatQuery = append(deployStatQuery, bson.M{"$limit": 5}) + deployStatQuery = append(deployStatQuery, bson.M{"$skip": 0}) + cursor, err := c.Aggregate(context.TODO(), deployStatQuery) + if err != nil { + return nil, err + } + if err := cursor.All(context.TODO(), &results); err != nil { + return nil, err + } + for _, result := range results { + deployItem := &models.DeployStat{ + ProductName: result.ID.ProductName, + MaxDeployFailureServiceName: result.ID.MaxDeployFailureServiceName, + MaxDeployFailureServiceNum: result.MaxDeployFailureServiceNum, + } + ret = append(ret, deployItem) + } + } + return ret, nil + } + + if len(option.ProductNames) > 0 { + query["product_name"] = bson.M{"$in": option.ProductNames} + } + + if option.StartDate > 0 { + query["create_time"] = bson.M{"$gte": option.StartDate, "$lte": option.EndDate} + } + + opt := &options.FindOptions{} + if option.IsAsc { + opt.SetSort(bson.D{{"create_time", 1}}) + } else { + opt.SetSort(bson.D{{"create_time", -1}}) + } + + cursor, err := c.Collection.Find(context.TODO(), query, opt) + if err != nil { + return nil, err + } + err = cursor.All(context.TODO(), &ret) + if err != nil { + return nil, err + } + return ret, nil +} + +func (c *DeployStatColl) GetDeployDailyTotal(args *models.DeployStatOption) ([]*DeployDailyItem, error) { + var result []*DeployDailyPipeResp + var resp []*DeployDailyItem + + pipeline := []bson.M{{ + "$sort": bson.M{ + "create_time": 1, + }, + }} + + if args.StartDate > 0 || args.EndDate > 0 { + timeRange := bson.M{} + if args.StartDate > 0 { + timeRange["$gte"] = args.StartDate + } + if args.EndDate > 0 { + timeRange["$lte"] = args.EndDate + } + pipeline = append(pipeline, bson.M{"$match": bson.M{"create_time": timeRange}}) + } + + pipeline = append(pipeline, bson.M{ + "$group": bson.M{ + "_id": bson.M{ + "date": "$date", + }, + "total_deploy_success": bson.M{ + "$sum": "$total_deploy_success", + }, + "total_deploy_failure": bson.M{ + "$sum": "$total_deploy_failure", + }, + }, + }) + + cursor, err := c.Aggregate(context.TODO(), pipeline) + if err != nil { + return nil, err + } + if err := cursor.All(context.TODO(), &result); err != nil { + return nil, err + } + for _, res := range result { + deployDailyItem := &DeployDailyItem{ + Date: res.ID.Date, + TotalSuccess: res.TotalSuccess, + TotalFailure: res.TotalFailure, + } + resp = append(resp, deployDailyItem) + } + + return resp, nil +} diff --git a/pkg/microservice/aslan/core/stat/repository/mongodb/test.go b/pkg/microservice/aslan/core/stat/repository/mongodb/test.go new file mode 100644 index 0000000000000000000000000000000000000000..04c16951ac3c1344738bd4c132b3ba30537d7fdd --- /dev/null +++ b/pkg/microservice/aslan/core/stat/repository/mongodb/test.go @@ -0,0 +1,118 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "errors" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/stat/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type TestStatOption struct { + StartDate int64 + EndDate int64 + IsAsc bool + ProductNames []string +} + +type TestStatColl struct { + *mongo.Collection + + coll string +} + +func NewTestStatColl() *TestStatColl { + name := models.TestStat{}.TableName() + return &TestStatColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} +} + +func (c *TestStatColl) GetCollectionName() string { + return c.coll +} + +func (c *TestStatColl) EnsureIndex(ctx context.Context) error { + mod := mongo.IndexModel{ + Keys: bson.D{ + bson.E{Key: "product_name", Value: 1}, + bson.E{Key: "date", Value: 1}, + }, + Options: options.Index().SetUnique(true), + } + + _, err := c.Indexes().CreateOne(ctx, mod) + + return err +} + +func (c *TestStatColl) Create(args *models.TestStat) error { + if args == nil { + return errors.New("nil testStat args") + } + + _, err := c.InsertOne(context.TODO(), args) + return err +} + +func (c *TestStatColl) FindCount() (int, error) { + cnt, err := c.Collection.CountDocuments(context.TODO(), bson.M{}) + return int(cnt), err +} + +func (c *TestStatColl) ListTestStat(option *TestStatOption) (testStats []*models.TestStat, err error) { + testStats = make([]*models.TestStat, 0) + query := bson.M{} + if len(option.ProductNames) > 0 { + query["product_name"] = bson.M{"$in": option.ProductNames} + } + + if option.StartDate > 0 { + query["create_time"] = bson.M{"$gte": option.StartDate, "$lte": option.EndDate} + } + + opt := &options.FindOptions{} + if option.IsAsc { + opt.SetSort(bson.D{{"create_time", 1}}) + } else { + opt.SetSort(bson.D{{"create_time", -1}}) + } + cursor, err := c.Collection.Find(context.TODO(), query, opt) + if err != nil { + return nil, err + } + err = cursor.All(context.TODO(), &testStats) + if err != nil { + return nil, err + } + return testStats, nil +} + +func (c *TestStatColl) Update(args *models.TestStat) error { + if args == nil { + return errors.New("nil testStat args") + } + + query := bson.M{"date": args.Date, "product_name": args.ProductName} + _, err := c.UpdateOne(context.TODO(), query, args) + return err +} diff --git a/pkg/microservice/aslan/core/stat/service/build.go b/pkg/microservice/aslan/core/stat/service/build.go new file mode 100644 index 0000000000000000000000000000000000000000..f589f1e2e5e6e239a65ff74389b09c5cfa0f1434 --- /dev/null +++ b/pkg/microservice/aslan/core/stat/service/build.go @@ -0,0 +1,77 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "sort" + + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/stat/repository/models" + repo "github.com/koderover/zadig/pkg/microservice/aslan/core/stat/repository/mongodb" +) + +type dashboardBuild struct { + Total int `json:"total"` + Success int `json:"success"` + DashboardBuildDailys []*dashboardBuildDaily `json:"data"` +} + +type dashboardBuildDaily struct { + Date string `json:"date"` + Success int `json:"success"` + Failure int `json:"failure"` + Total int `json:"total"` +} + +func GetBuildTotalAndSuccess(args *models.BuildStatOption, log *zap.SugaredLogger) (*dashboardBuild, error) { + var ( + dashboardBuild = new(dashboardBuild) + dashboardBuildDailys = make([]*dashboardBuildDaily, 0) + total = 0 + success = 0 + ) + if buildItems, err := repo.NewBuildStatColl().GetBuildTotalAndSuccess(); err == nil { + for _, buildItem := range buildItems { + success += buildItem.TotalSuccess + total += buildItem.TotalBuildCount + } + dashboardBuild.Success = success + dashboardBuild.Total = total + } else { + log.Errorf("Failed to getBuildTotalAndSuccess err:%s", err) + return nil, err + } + + if buildDailyItems, err := repo.NewBuildStatColl().GetBuildDailyTotal(args); err == nil { + sort.SliceStable(buildDailyItems, func(i, j int) bool { return buildDailyItems[i].Date < buildDailyItems[j].Date }) + for _, buildDailyItem := range buildDailyItems { + dashboardBuildDaily := new(dashboardBuildDaily) + dashboardBuildDaily.Date = buildDailyItem.Date + dashboardBuildDaily.Success = buildDailyItem.TotalSuccess + dashboardBuildDaily.Failure = buildDailyItem.TotalFailure + dashboardBuildDaily.Total = buildDailyItem.TotalBuildCount + + dashboardBuildDailys = append(dashboardBuildDailys, dashboardBuildDaily) + } + dashboardBuild.DashboardBuildDailys = dashboardBuildDailys + } else { + log.Errorf("Failed to getDeployDailyTotal err:%s", err) + return nil, err + } + return dashboardBuild, nil +} diff --git a/pkg/microservice/aslan/core/stat/service/build_stat.go b/pkg/microservice/aslan/core/stat/service/build_stat.go new file mode 100644 index 0000000000000000000000000000000000000000..98dff3ea935582d2ad9cb33499dc1c2947e56229 --- /dev/null +++ b/pkg/microservice/aslan/core/stat/service/build_stat.go @@ -0,0 +1,486 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "fmt" + "math" + "sort" + "time" + + "github.com/jinzhu/now" + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + taskmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" + commonmongodb "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" + "github.com/koderover/zadig/pkg/microservice/aslan/core/stat/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/stat/repository/mongodb" +) + +func GetAllPipelineTask(log *zap.SugaredLogger) error { + option := &commonmongodb.ListAllTaskOption{Type: config.WorkflowType} + count, err := mongodb.NewBuildStatColl().FindCount() + if err != nil { + log.Errorf("BuildStat FindCount err:%v", err) + return fmt.Errorf("BuildStat FindCount err:%v", err) + } + if count > 0 { + option.CreateTime = time.Now().AddDate(0, 0, -1).Unix() + } + //获取所有的项目名称 + allProducts, err := templaterepo.NewProductColl().List() + if err != nil { + log.Errorf("BuildStat ProductTmpl List err:%v", err) + return fmt.Errorf("BuildStat ProductTmpl List err:%v", err) + } + for _, product := range allProducts { + option.ProductNames = []string{product.ProductName} + allTasks, err := commonmongodb.NewTaskColl().ListAllTasks(option) + if err != nil { + log.Errorf("pipeline list err:%v", err) + return fmt.Errorf("pipeline list err:%v", err) + } + taskDateMap := make(map[string][]*taskmodels.Task) + if len(allTasks) > 0 { + //将task的时间戳转成日期，以日期为单位分组 + for _, task := range allTasks { + time := time.Unix(task.CreateTime, 0) + date := time.Format(config.Date) + if _, isExist := taskDateMap[date]; isExist { + taskDateMap[date] = append(taskDateMap[date], task) + } else { + tasks := make([]*taskmodels.Task, 0) + tasks = append(tasks, task) + taskDateMap[date] = tasks + } + } + } else { + localTime := time.Now().AddDate(0, 0, -1).In(time.Local) + date := localTime.Format(config.Date) + taskDateMap[date] = []*taskmodels.Task{ + {Stages: []*commonmodels.Stage{}}, + } + } + + taskDateKeys := make([]string, 0, len(taskDateMap)) + for taskDateMapKey := range taskDateMap { + taskDateKeys = append(taskDateKeys, taskDateMapKey) + } + sort.Strings(taskDateKeys) + + for _, taskDate := range taskDateKeys { + var ( + totalSuccess = 0 + totalFailure = 0 + totalTimeout = 0 + totalDuration int64 + maxDurationPipelines = make([]*models.PipelineInfo, 0) + ) + //循环task任务获取需要的数据 + for _, taskPreview := range taskDateMap[taskDate] { + stages := taskPreview.Stages + for _, subStage := range stages { + taskType := subStage.TaskType + switch taskType { + case config.TaskBuild: + // 获取构建时长 + for _, subTask := range subStage.SubTasks { + buildInfo, err := base.ToBuildTask(subTask) + if err != nil { + log.Errorf("BuildStat ToBuildTask err:%v", err) + continue + } + if buildInfo.TaskStatus == config.StatusPassed { + totalSuccess++ + } else if buildInfo.TaskStatus == config.StatusFailed { + totalFailure++ + } else if buildInfo.TaskStatus == config.StatusTimeout { + totalTimeout++ + } else { + continue + } + + totalDuration += buildInfo.EndTime - buildInfo.StartTime + maxDurationPipeline := new(models.PipelineInfo) + maxDurationPipeline.PipelineName = taskPreview.PipelineName + maxDurationPipeline.TaskID = taskPreview.TaskID + maxDurationPipeline.Type = string(taskPreview.Type) + maxDurationPipeline.MaxDuration = buildInfo.EndTime - buildInfo.StartTime + + maxDurationPipelines = append(maxDurationPipelines, maxDurationPipeline) + } + } + } + } + //比较maxDurationPipeline的MaxDuration的最大值 + sort.SliceStable(maxDurationPipelines, func(i, j int) bool { return maxDurationPipelines[i].MaxDuration > maxDurationPipelines[j].MaxDuration }) + buildStat := new(models.BuildStat) + buildStat.ProductName = product.ProductName + buildStat.TotalSuccess = totalSuccess + buildStat.TotalFailure = totalFailure + buildStat.TotalTimeout = totalTimeout + buildStat.TotalDuration = totalDuration + if len(maxDurationPipelines) > 0 { + buildStat.TotalBuildCount = len(maxDurationPipelines) + buildStat.MaxDuration = maxDurationPipelines[0].MaxDuration + buildStat.MaxDurationPipeline = maxDurationPipelines[0] + } else { + buildStat.TotalBuildCount = 0 + buildStat.MaxDuration = 0 + buildStat.MaxDurationPipeline = &models.PipelineInfo{} + } + buildStat.Date = taskDate + tt, _ := time.ParseInLocation(config.Date, taskDate, time.Local) + buildStat.CreateTime = tt.Unix() + buildStat.UpdateTime = time.Now().Unix() + + err := mongodb.NewBuildStatColl().Create(buildStat) + if err != nil { //插入失败就更新 + err = mongodb.NewBuildStatColl().Update(buildStat) + if err != nil { + log.Errorf("BuildStat Update err:%v", err) + continue + } + } + } + } + return nil +} + +type dailyBuildStat struct { + Date string `bson:"date" json:"date"` + AverageDuration int `bson:"average_duration" json:"averageDuration"` +} + +func GetBuildDailyAverageMeasure(startDate int64, endDate int64, productNames []string, log *zap.SugaredLogger) ([]*dailyBuildStat, error) { + buildStats, err := mongodb.NewBuildStatColl().ListBuildStat(&models.BuildStatOption{StartDate: startDate, EndDate: endDate, IsAsc: true, ProductNames: productNames}) + if err != nil { + log.Errorf("ListBuildStat err:%v", err) + return nil, fmt.Errorf("ListBuildStat err:%v", err) + } + buildStatMap := make(map[string][]*models.BuildStat) + for _, buildStat := range buildStats { + if _, isExist := buildStatMap[buildStat.Date]; isExist { + buildStatMap[buildStat.Date] = append(buildStatMap[buildStat.Date], buildStat) + } else { + tempBuildStats := make([]*models.BuildStat, 0) + tempBuildStats = append(tempBuildStats, buildStat) + buildStatMap[buildStat.Date] = tempBuildStats + } + } + buildStatDateKeys := make([]string, 0, len(buildStatMap)) + for buildStatDateMapKey := range buildStatMap { + buildStatDateKeys = append(buildStatDateKeys, buildStatDateMapKey) + } + sort.Strings(buildStatDateKeys) + buildStatDailyArgs := make([]*dailyBuildStat, 0) + for _, buildStatDateKey := range buildStatDateKeys { + totalDuration := 0 + totalBuildCount := 0 + for _, buildStat := range buildStatMap[buildStatDateKey] { + totalDuration += int(buildStat.TotalDuration) + totalBuildCount += buildStat.TotalBuildCount + } + buildStatDailyArg := new(dailyBuildStat) + buildStatDailyArg.Date = buildStatDateKey + if totalBuildCount > 0 { + buildStatDailyArg.AverageDuration = int(math.Floor(float64(totalDuration)/float64(totalBuildCount) + 0.5)) + } + + buildStatDailyArgs = append(buildStatDailyArgs, buildStatDailyArg) + } + return buildStatDailyArgs, nil +} + +type buildStatDaily struct { + Date string `bson:"date" json:"date"` + Success int `bson:"success" json:"success"` + Failure int `bson:"failure" json:"failure"` +} + +func GetBuildDailyMeasure(startDate int64, endDate int64, productNames []string, log *zap.SugaredLogger) ([]*buildStatDaily, error) { + buildStats, err := mongodb.NewBuildStatColl().ListBuildStat(&models.BuildStatOption{StartDate: startDate, EndDate: endDate, IsAsc: true, ProductNames: productNames}) + if err != nil { + log.Errorf("ListBuildStat err:%v", err) + return nil, fmt.Errorf("ListBuildStat err:%v", err) + } + + buildStatMap := make(map[string][]*models.BuildStat) + for _, buildStat := range buildStats { + if _, isExist := buildStatMap[buildStat.Date]; isExist { + buildStatMap[buildStat.Date] = append(buildStatMap[buildStat.Date], buildStat) + } else { + tempBuildStats := make([]*models.BuildStat, 0) + tempBuildStats = append(tempBuildStats, buildStat) + buildStatMap[buildStat.Date] = tempBuildStats + } + + } + buildStatDateKeys := make([]string, 0, len(buildStatMap)) + for buildStatDateMapKey := range buildStatMap { + buildStatDateKeys = append(buildStatDateKeys, buildStatDateMapKey) + } + sort.Strings(buildStatDateKeys) + + buildStatDailys := make([]*buildStatDaily, 0) + for _, buildStatDateKey := range buildStatDateKeys { + totalSuccess := 0 + totalFailure := 0 + for _, buildStat := range buildStatMap[buildStatDateKey] { + totalSuccess += buildStat.TotalSuccess + totalFailure += buildStat.TotalFailure + } + + buildStatDaily := new(buildStatDaily) + buildStatDaily.Date = buildStatDateKey + buildStatDaily.Success = totalSuccess + buildStatDaily.Failure = totalFailure + + buildStatDailys = append(buildStatDailys, buildStatDaily) + } + return buildStatDailys, nil +} + +type buildStatTotal struct { + TotalSuccess int `json:"totalSuccess"` + TotalFailure int `bson:"total_failure" json:"totalFailure"` +} + +func GetBuildHealthMeasure(startDate int64, endDate int64, productNames []string, log *zap.SugaredLogger) (*buildStatTotal, error) { + buildStats, err := mongodb.NewBuildStatColl().ListBuildStat(&models.BuildStatOption{StartDate: startDate, EndDate: endDate, IsAsc: true, ProductNames: productNames}) + if err != nil { + log.Errorf("ListBuildStat err:%v", err) + return nil, fmt.Errorf("ListBuildStat err:%v", err) + } + var ( + totalSuccess = 0 + totalFailure = 0 + ) + for _, buildStat := range buildStats { + totalSuccess += buildStat.TotalSuccess + totalFailure += buildStat.TotalFailure + } + buildStatTotal := &buildStatTotal{ + TotalSuccess: totalSuccess, + TotalFailure: totalFailure, + } + return buildStatTotal, nil +} + +type buildStatLatestTen struct { + *models.PipelineInfo + + ProductName string `bson:"product_name" json:"productName"` + Duration int64 `bson:"duration" json:"duration"` + Status string `bson:"status" json:"status"` + TaskCreator string `bson:"task_creator" json:"taskCreator"` + CreateTime int64 `bson:"create_time" json:"createTime"` +} + +func GetLatestTenBuildMeasure(productNames []string, log *zap.SugaredLogger) ([]*buildStatLatestTen, error) { + latestTenTasks, err := commonmongodb.NewTaskColl().ListAllTasks(&commonmongodb.ListAllTaskOption{Type: config.WorkflowType, Limit: config.LatestDay, Skip: 0, ProductNames: productNames}) + if err != nil { + log.Errorf("pipeline ListAllTasks err:%v", err) + return nil, fmt.Errorf("pipeline ListAllTasks err:%v", err) + } + latestTenPipelines := make([]*buildStatLatestTen, 0) + for _, latestTask := range latestTenTasks { + buildStatLatestTen := &buildStatLatestTen{ + PipelineInfo: &models.PipelineInfo{ + TaskID: latestTask.TaskID, + Type: string(latestTask.Type), + PipelineName: latestTask.PipelineName, + }, + ProductName: latestTask.ProductName, + Duration: latestTask.EndTime - latestTask.StartTime, + Status: string(latestTask.Status), + TaskCreator: latestTask.TaskCreator, + CreateTime: latestTask.CreateTime, + } + latestTenPipelines = append(latestTenPipelines, buildStatLatestTen) + } + + return latestTenPipelines, nil +} + +func GetTenDurationMeasure(startDate int64, endDate int64, productNames []string, log *zap.SugaredLogger) ([]*buildStatLatestTen, error) { + maxTenDurationBuildStats, err := mongodb.NewBuildStatColl().ListBuildStat(&models.BuildStatOption{StartDate: startDate, EndDate: endDate, Limit: config.LatestDay, Skip: 0, ProductNames: productNames}) + if err != nil { + log.Errorf("ListBuildStat err:%v", err) + return nil, fmt.Errorf("ListBuildStat err:%v", err) + } + latestTenPipelines := make([]*buildStatLatestTen, 0) + for _, buidStat := range maxTenDurationBuildStats { + task, err := commonmongodb.NewTaskColl().Find(buidStat.MaxDurationPipeline.TaskID, buidStat.MaxDurationPipeline.PipelineName, config.PipelineType(buidStat.MaxDurationPipeline.Type)) + if err != nil { + log.Errorf("PipelineTask Find err:%v", err) + continue + } + + buildStatLatestTen := &buildStatLatestTen{ + PipelineInfo: buidStat.MaxDurationPipeline, + ProductName: task.ProductName, + Duration: task.EndTime - task.StartTime, + Status: string(task.Status), + TaskCreator: task.TaskCreator, + CreateTime: task.CreateTime, + } + latestTenPipelines = append(latestTenPipelines, buildStatLatestTen) + } + return latestTenPipelines, nil +} + +type buildTrend struct { + *CurrentDay + Sum []*sumData `json:"sum"` +} + +type CurrentDay struct { + Success int `json:"success"` + Failure int `json:"failure"` + Timeout int `json:"timeout"` +} + +type sumData struct { + *CurrentDay + Day int64 `json:"day"` +} + +func GetBuildTrendMeasure(startDate int64, endDate int64, productNames []string, log *zap.SugaredLogger) (*buildTrend, error) { + dayStartTimestamp := now.BeginningOfDay().Unix() + var ( + totalSuccess = 0 + totalFailure = 0 + totalTimeout = 0 + ) + allTasks, err := commonmongodb.NewTaskColl().ListAllTasks(&commonmongodb.ListAllTaskOption{Type: config.WorkflowType, CreateTime: dayStartTimestamp, ProductNames: productNames}) + if err != nil { + log.Errorf("ListAllTasks err:%v", err) + return nil, fmt.Errorf("ListAllTasks err:%v", err) + } + for _, taskPreview := range allTasks { + stages := taskPreview.Stages + for _, subStage := range stages { + taskType := subStage.TaskType + switch taskType { + case config.TaskBuild: + // 获取构建时长 + for _, subTask := range subStage.SubTasks { + buildInfo, err := base.ToBuildTask(subTask) + if err != nil { + log.Errorf("BuildStat ToBuildTask err:%v", err) + continue + } + if buildInfo.TaskStatus == config.StatusPassed { + totalSuccess++ + } else if buildInfo.TaskStatus == config.StatusFailed { + totalFailure++ + } else if buildInfo.TaskStatus == config.StatusTimeout { + totalTimeout++ + } else { + continue + } + } + } + } + } + curr := &CurrentDay{ + Success: totalSuccess, + Failure: totalFailure, + Timeout: totalTimeout, + } + sumDatas := make([]*sumData, 0) + buildStats, err := mongodb.NewBuildStatColl().ListBuildStat(&models.BuildStatOption{StartDate: startDate, EndDate: endDate, IsAsc: false, ProductNames: productNames}) + if err != nil { + log.Errorf("ListBuildStat err:%v", err) + return nil, fmt.Errorf("ListBuildStat err:%v", err) + } + buildStatMap := make(map[string][]*models.BuildStat) + for _, buildStat := range buildStats { + if _, isExist := buildStatMap[buildStat.Date]; isExist { + buildStatMap[buildStat.Date] = append(buildStatMap[buildStat.Date], buildStat) + } else { + tempBuildStats := make([]*models.BuildStat, 0) + tempBuildStats = append(tempBuildStats, buildStat) + buildStatMap[buildStat.Date] = tempBuildStats + } + } + buildStatDateKeys := make([]string, 0, len(buildStatMap)) + for buildStatDateMapKey := range buildStatMap { + buildStatDateKeys = append(buildStatDateKeys, buildStatDateMapKey) + } + sort.Sort(sort.Reverse(sort.StringSlice(buildStatDateKeys))) + + totalSuccess = 0 + totalFailure = 0 + totalTimeout = 0 + if len(buildStatMap) <= config.Day { + for _, buildStat := range buildStats { + totalSuccess += buildStat.TotalSuccess + totalFailure += buildStat.TotalFailure + totalTimeout += buildStat.TotalTimeout + } + sumData := &sumData{ + Day: time.Now().Unix(), + CurrentDay: &CurrentDay{ + Success: totalSuccess, + Failure: totalFailure, + Timeout: totalTimeout, + }, + } + sumDatas = append(sumDatas, sumData) + } else { + for index, buildStatDate := range buildStatDateKeys { + for _, buildStat := range buildStatMap[buildStatDate] { + totalSuccess += buildStat.TotalSuccess + totalFailure += buildStat.TotalFailure + totalTimeout += buildStat.TotalTimeout + } + + if ((index + 1) % config.Day) == 0 { + date, err := time.ParseInLocation(config.Date, buildStatDate, time.Local) + if err != nil { + log.Errorf("Failed to parse date: %s, the error is: %+v", buildStatDate, err) + return nil, err + } + sumData := &sumData{ + Day: date.Unix(), + CurrentDay: &CurrentDay{ + Success: totalSuccess, + Failure: totalFailure, + Timeout: totalTimeout, + }, + } + sumDatas = append(sumDatas, sumData) + totalSuccess = 0 + totalFailure = 0 + totalTimeout = 0 + } + } + } + + buildTrend := &buildTrend{ + CurrentDay: curr, + Sum: sumDatas, + } + + return buildTrend, nil +} diff --git a/pkg/microservice/aslan/core/stat/service/deploy.go b/pkg/microservice/aslan/core/stat/service/deploy.go new file mode 100644 index 0000000000000000000000000000000000000000..7051262c9a34af29836f35ba1802b5dff90a7c8a --- /dev/null +++ b/pkg/microservice/aslan/core/stat/service/deploy.go @@ -0,0 +1,79 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "sort" + + "go.uber.org/zap" + + models "github.com/koderover/zadig/pkg/microservice/aslan/core/stat/repository/models" + repo "github.com/koderover/zadig/pkg/microservice/aslan/core/stat/repository/mongodb" +) + +type dashboardDeploy struct { + Total int `json:"total"` + Success int `json:"success"` + DashboardDeployDailys []*dashboardDeployDaily `json:"data"` +} + +type dashboardDeployDaily struct { + Date string `json:"date"` + Success int `json:"success"` + Failure int `json:"failure"` + Total int `json:"total"` +} + +func GetDeployDailyTotalAndSuccess(args *models.DeployStatOption, log *zap.SugaredLogger) (*dashboardDeploy, error) { + var ( + dashboardDeploy = new(dashboardDeploy) + dashboardDeployDailys = make([]*dashboardDeployDaily, 0) + failure int + success int + ) + + if deployItems, err := repo.NewDeployStatColl().GetDeployTotalAndSuccess(); err == nil { + for _, deployItem := range deployItems { + success += deployItem.TotalSuccess + failure += deployItem.TotalFailure + } + dashboardDeploy.Total = success + failure + dashboardDeploy.Success = success + } else { + log.Errorf("Failed to getDeployTotalAndSuccess err:%s", err) + return nil, err + } + + if deployDailyItems, err := repo.NewDeployStatColl().GetDeployDailyTotal(args); err == nil { + sort.SliceStable(deployDailyItems, func(i, j int) bool { return deployDailyItems[i].Date < deployDailyItems[j].Date }) + for _, deployDailyItem := range deployDailyItems { + dashboardDeployDaily := new(dashboardDeployDaily) + dashboardDeployDaily.Date = deployDailyItem.Date + dashboardDeployDaily.Success = deployDailyItem.TotalSuccess + dashboardDeployDaily.Failure = deployDailyItem.TotalFailure + dashboardDeployDaily.Total = deployDailyItem.TotalFailure + deployDailyItem.TotalSuccess + + dashboardDeployDailys = append(dashboardDeployDailys, dashboardDeployDaily) + } + dashboardDeploy.DashboardDeployDailys = dashboardDeployDailys + } else { + log.Errorf("Failed to getDeployDailyTotal err:%s", err) + return nil, err + } + + return dashboardDeploy, nil +} diff --git a/pkg/microservice/aslan/core/stat/service/deploy_stat.go b/pkg/microservice/aslan/core/stat/service/deploy_stat.go new file mode 100644 index 0000000000000000000000000000000000000000..ac63b6d2580825457a9960f75e4b5a17e2a2f0ec --- /dev/null +++ b/pkg/microservice/aslan/core/stat/service/deploy_stat.go @@ -0,0 +1,395 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "fmt" + "sort" + "time" + + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + taskmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" + commonmongodb "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" + "github.com/koderover/zadig/pkg/microservice/aslan/core/stat/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/stat/repository/mongodb" +) + +type serviceInfo struct { + ServiceName string + DeploySuccess int + DeployFailure int +} + +type serviceTotalInfo struct { + ServiceName string + DeployTotal int + DeployTotalFailure int +} + +func InitDeployStat(log *zap.SugaredLogger) error { + option := &commonmongodb.ListAllTaskOption{Type: config.WorkflowType} + count, err := mongodb.NewDeployStatColl().FindCount() + if err != nil { + log.Errorf("deployStat FindCount err:%v", err) + return fmt.Errorf("deployStat FindCount err:%v", err) + } + if count > 0 { + option.CreateTime = time.Now().AddDate(0, 0, -1).Unix() + } + //获取所有的项目名称 + allProducts, err := templaterepo.NewProductColl().List() + if err != nil { + log.Errorf("deployStat ProductTmpl List err:%v", err) + return fmt.Errorf("deployStat ProductTmpl List err:%v", err) + } + for _, product := range allProducts { + option.ProductNames = []string{product.ProductName} + allTasks, err := commonmongodb.NewTaskColl().ListAllTasks(option) + if err != nil { + log.Errorf("pipeline list err:%v", err) + return fmt.Errorf("pipeline list err:%v", err) + } + taskDateMap := make(map[string][]*taskmodels.Task) + if len(allTasks) > 0 { + //将task的时间戳转成日期，以日期为单位分组 + for _, task := range allTasks { + createTime := time.Unix(task.CreateTime, 0) + date := createTime.Format(config.Date) + if _, isExist := taskDateMap[date]; isExist { + taskDateMap[date] = append(taskDateMap[date], task) + } else { + tasks := make([]*taskmodels.Task, 0) + tasks = append(tasks, task) + taskDateMap[date] = tasks + } + } + } else { + currentTime := time.Now().AddDate(0, 0, -1).In(time.Local) + date := currentTime.Format(config.Date) + taskDateMap[date] = []*taskmodels.Task{ + {Stages: []*commonmodels.Stage{}}, + } + } + + taskDateKeys := make([]string, 0, len(taskDateMap)) + for taskDateMapKey := range taskDateMap { + taskDateKeys = append(taskDateKeys, taskDateMapKey) + } + sort.Strings(taskDateKeys) + + for _, taskDate := range taskDateKeys { + var ( + totalTaskSuccess = 0 + totalTaskFailure = 0 + totalDeploySuccess = 0 + totalDeployFailure = 0 + deployServiceInfos = make([]*serviceInfo, 0) + maxDeployFailureServiceMap = make(map[string][]*serviceInfo) + serviceInfoTotals = make([]*serviceTotalInfo, 0) + ) + //循环task任务获取需要的数据 + for _, taskPreview := range taskDateMap[taskDate] { + stages := taskPreview.Stages + taskStatus := taskPreview.Status + switch taskStatus { + case config.StatusPassed: + totalTaskSuccess++ + case config.StatusFailed: + totalTaskFailure++ + } + for _, subStage := range stages { + taskType := subStage.TaskType + switch taskType { + case config.TaskDeploy: + for _, subTask := range subStage.SubTasks { + deployInfo, err := base.ToDeployTask(subTask) + serviceInfo := new(serviceInfo) + serviceInfo.ServiceName = deployInfo.ServiceName + + if err != nil { + log.Errorf("deployStat ToDeployTask err:%v", err) + continue + } + + if deployInfo.TaskStatus == config.StatusPassed { + totalDeploySuccess++ + serviceInfo.DeploySuccess = 1 + } else if deployInfo.TaskStatus == config.StatusFailed { + totalDeployFailure++ + serviceInfo.DeployFailure = 1 + } else { + continue + } + deployServiceInfos = append(deployServiceInfos, serviceInfo) + } + } + } + } + //以服务名称分组 + for _, svcInfo := range deployServiceInfos { + if _, isExsit := maxDeployFailureServiceMap[svcInfo.ServiceName]; isExsit { + maxDeployFailureServiceMap[svcInfo.ServiceName] = append(maxDeployFailureServiceMap[svcInfo.ServiceName], svcInfo) + } else { + serviceInfos := make([]*serviceInfo, 0) + serviceInfos = append(serviceInfos, svcInfo) + maxDeployFailureServiceMap[svcInfo.ServiceName] = serviceInfos + } + } + //统计部署次数最高和失败最高的服务 + for serviceName, serviceInfos := range maxDeployFailureServiceMap { + totalDeploy := 0 + totalFailure := 0 + for _, serviceInfo := range serviceInfos { + totalFailure += serviceInfo.DeployFailure + totalDeploy += totalFailure + serviceInfo.DeploySuccess + } + serviceTotalInfo := &serviceTotalInfo{ + ServiceName: serviceName, + DeployTotal: totalDeploy, + DeployTotalFailure: totalFailure, + } + serviceInfoTotals = append(serviceInfoTotals, serviceTotalInfo) + } + + deployStat := new(models.DeployStat) + deployStat.ProductName = product.ProductName + deployStat.TotalTaskSuccess = totalTaskSuccess + deployStat.TotalTaskFailure = totalTaskFailure + deployStat.TotalDeploySuccess = totalDeploySuccess + deployStat.TotalDeployFailure = totalDeployFailure + deployStat.Date = taskDate + tt, _ := time.ParseInLocation(config.Date, taskDate, time.Local) + deployStat.CreateTime = tt.Unix() + deployStat.UpdateTime = time.Now().Unix() + if len(serviceInfoTotals) > 0 { + sort.SliceStable(serviceInfoTotals, func(i, j int) bool { return serviceInfoTotals[i].DeployTotal > serviceInfoTotals[j].DeployTotal }) + deployStat.MaxDeployServiceNum = serviceInfoTotals[0].DeployTotal + deployStat.MaxDeployServiceFailureNum = serviceInfoTotals[0].DeployTotalFailure + deployStat.MaxDeployServiceName = serviceInfoTotals[0].ServiceName + + sort.SliceStable(serviceInfoTotals, func(i, j int) bool { + return serviceInfoTotals[i].DeployTotalFailure > serviceInfoTotals[j].DeployTotalFailure + }) + deployStat.MaxDeployFailureServiceNum = serviceInfoTotals[0].DeployTotalFailure + deployStat.MaxDeployFailureServiceName = serviceInfoTotals[0].ServiceName + } else { + deployStat.MaxDeployServiceNum = 0 + deployStat.MaxDeployServiceFailureNum = 0 + deployStat.MaxDeployServiceName = "" + deployStat.MaxDeployFailureServiceNum = 0 + deployStat.MaxDeployFailureServiceName = "" + } + err := mongodb.NewDeployStatColl().Create(deployStat) + if err != nil { //插入失败就更新 + err = mongodb.NewDeployStatColl().Update(deployStat) + if err != nil { + log.Errorf("deployStat Update err:%v", err) + continue + } + } + } + } + return nil +} + +type deployStatTotal struct { + TotalSuccess int `bson:"total_success" json:"totalSuccess"` + TotalFailure int `bson:"total_failure" json:"totalFailure"` +} + +func GetPipelineHealthMeasure(startDate, endDate int64, productNames []string, log *zap.SugaredLogger) (*deployStatTotal, error) { + deployStats, err := mongodb.NewDeployStatColl().ListDeployStat(&models.DeployStatOption{StartDate: startDate, EndDate: endDate, IsAsc: true, ProductNames: productNames}) + if err != nil { + log.Errorf("ListDeployStat err:%v", err) + return nil, fmt.Errorf("ListDeployStat err:%v", err) + } + var ( + totalSuccess = 0 + totalFailure = 0 + ) + for _, deployStat := range deployStats { + totalSuccess += deployStat.TotalTaskSuccess + totalFailure += deployStat.TotalTaskFailure + } + deployStatTotal := &deployStatTotal{ + TotalSuccess: totalSuccess, + TotalFailure: totalFailure, + } + return deployStatTotal, nil +} + +func GetDeployHealthMeasure(startDate, endDate int64, productNames []string, log *zap.SugaredLogger) (*deployStatTotal, error) { + deployStats, err := mongodb.NewDeployStatColl().ListDeployStat(&models.DeployStatOption{StartDate: startDate, EndDate: endDate, IsAsc: true, ProductNames: productNames}) + if err != nil { + log.Errorf("ListDeployStat err:%v", err) + return nil, fmt.Errorf("ListDeployStat err:%v", err) + } + var ( + totalSuccess = 0 + totalFailure = 0 + ) + for _, deployStat := range deployStats { + totalSuccess += deployStat.TotalDeploySuccess + totalFailure += deployStat.TotalDeployFailure + } + deployStatTotal := &deployStatTotal{ + TotalSuccess: totalSuccess, + TotalFailure: totalFailure, + } + return deployStatTotal, nil +} + +type deployStatWeekly struct { + Day int64 `bson:"day" json:"day"` + TotalSuccess int `bson:"total_success" json:"totalSuccess"` + TotalFailure int `bson:"total_failure" json:"totalFailure"` +} + +func GetDeployWeeklyMeasure(startDate, endDate int64, productNames []string, log *zap.SugaredLogger) ([]*deployStatWeekly, error) { + deployStats, err := mongodb.NewDeployStatColl().ListDeployStat(&models.DeployStatOption{StartDate: startDate, EndDate: endDate, IsAsc: true, ProductNames: productNames}) + if err != nil { + log.Errorf("ListDeployStat err:%v", err) + return nil, fmt.Errorf("ListDeployStat err:%v", err) + } + deployStatMap := make(map[string][]*models.DeployStat) + for _, deployStat := range deployStats { + if _, isExist := deployStatMap[deployStat.Date]; isExist { + deployStatMap[deployStat.Date] = append(deployStatMap[deployStat.Date], deployStat) + } else { + tempDeployStats := make([]*models.DeployStat, 0) + tempDeployStats = append(tempDeployStats, deployStat) + deployStatMap[deployStat.Date] = tempDeployStats + } + } + deployStatDateKeys := make([]string, 0, len(deployStatMap)) + for deployStatDateMapKey := range deployStatMap { + deployStatDateKeys = append(deployStatDateKeys, deployStatDateMapKey) + } + sort.Sort(sort.Reverse(sort.StringSlice(deployStatDateKeys))) + + var ( + totalSuccess = 0 + totalFailure = 0 + ) + deployStatWeeklys := make([]*deployStatWeekly, 0) + if len(deployStatDateKeys) <= config.Day && len(deployStatDateKeys) > 0 { + for _, deployStat := range deployStats { + totalSuccess += deployStat.TotalDeploySuccess + totalFailure += deployStat.TotalDeployFailure + } + deployStatWeekly := &deployStatWeekly{ + Day: time.Now().Unix(), + TotalSuccess: totalSuccess, + TotalFailure: totalFailure, + } + deployStatWeeklys = append(deployStatWeeklys, deployStatWeekly) + } else { + for index, deployStatDate := range deployStatDateKeys { + for _, deployStat := range deployStatMap[deployStatDate] { + totalSuccess += deployStat.TotalDeploySuccess + totalFailure += deployStat.TotalDeployFailure + } + + if ((index + 1) % config.Day) == 0 { + date, err := time.ParseInLocation(config.Date, deployStatDate, time.Local) + if err != nil { + log.Errorf("Failed to parse date: %s, the error is: %+v", deployStatDate, err) + return nil, err + } + deployStatWeekly := &deployStatWeekly{ + Day: date.Unix(), + TotalSuccess: totalSuccess, + TotalFailure: totalFailure, + } + deployStatWeeklys = append(deployStatWeeklys, deployStatWeekly) + totalSuccess = 0 + totalFailure = 0 + } + } + } + + return deployStatWeeklys, nil +} + +type deployHigherStat struct { + ServiceName string `bson:"service_name" json:"serviceName"` + TotalSuccess int `bson:"total_success" json:"totalSuccess"` + TotalFailure int `bson:"total_failure" json:"totalFailure"` +} + +func GetDeployTopFiveHigherMeasure(startDate, endDate int64, productNames []string, log *zap.SugaredLogger) ([]*deployHigherStat, error) { + deployStats, err := mongodb.NewDeployStatColl().ListDeployStat(&models.DeployStatOption{StartDate: startDate, EndDate: endDate, IsAsc: true, ProductNames: productNames, Limit: 5, IsMaxDeploy: true}) + if err != nil { + log.Errorf("ListDeployStat err:%v", err) + return nil, fmt.Errorf("ListDeployStat err:%v", err) + } + deployHigherStats := make([]*deployHigherStat, 0) + for _, deployStat := range deployStats { + tempDeployStat, err := mongodb.NewDeployStatColl().Get(&mongodb.DeployStatGetOption{ServiceName: deployStat.MaxDeployServiceName, MaxDeployServiceNum: deployStat.MaxDeployServiceNum}) + if err != nil { + log.Errorf("Get deployStat err:%v", err) + continue + } + deployHigherStat := &deployHigherStat{ + ServiceName: tempDeployStat.MaxDeployServiceName, + TotalSuccess: tempDeployStat.MaxDeployServiceNum - tempDeployStat.MaxDeployServiceFailureNum, + TotalFailure: tempDeployStat.MaxDeployServiceFailureNum, + } + if tempDeployStat.MaxDeployServiceName != "" { + deployHigherStats = append(deployHigherStats, deployHigherStat) + } + } + sort.SliceStable(deployHigherStats, func(i, j int) bool { + return deployHigherStats[i].TotalSuccess > deployHigherStats[j].TotalSuccess + }) + return deployHigherStats, nil +} + +type deployFailureHigherStat struct { + ProductName string `bson:"product_name" json:"productName"` + ServiceName string `bson:"service_name" json:"serviceName"` + TotalFailure int `bson:"total_failure" json:"totalFailure"` +} + +func GetDeployTopFiveFailureMeasure(startDate, endDate int64, productNames []string, log *zap.SugaredLogger) ([]*deployFailureHigherStat, error) { + deployStats, err := mongodb.NewDeployStatColl().ListDeployStat(&models.DeployStatOption{StartDate: startDate, EndDate: endDate, IsAsc: true, ProductNames: productNames, Limit: 5}) + if err != nil { + log.Errorf("ListDeployStat err:%v", err) + return nil, fmt.Errorf("ListDeployStat err:%v", err) + } + deployFailureHigherStats := make([]*deployFailureHigherStat, 0) + for _, deployStat := range deployStats { + deployFailureHigherStat := &deployFailureHigherStat{ + ProductName: deployStat.ProductName, + ServiceName: deployStat.MaxDeployFailureServiceName, + TotalFailure: deployStat.MaxDeployFailureServiceNum, + } + if deployStat.MaxDeployFailureServiceName != "" { + deployFailureHigherStats = append(deployFailureHigherStats, deployFailureHigherStat) + } + } + + sort.SliceStable(deployFailureHigherStats, func(i, j int) bool { + return deployFailureHigherStats[i].TotalFailure > deployFailureHigherStats[j].TotalFailure + }) + return deployFailureHigherStats, nil +} diff --git a/pkg/microservice/aslan/core/stat/service/overview.go b/pkg/microservice/aslan/core/stat/service/overview.go new file mode 100644 index 0000000000000000000000000000000000000000..cee8f27336943f4b40e0ac6e00b706b439458821 --- /dev/null +++ b/pkg/microservice/aslan/core/stat/service/overview.go @@ -0,0 +1,80 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "go.uber.org/zap" + + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" +) + +type Overview struct { + ProjectCount int `json:"project_count"` + ClusterCount int `json:"cluster_count"` + ServiceCount int `json:"service_count"` + WorkflowCount int `json:"workflow_count"` + EnvCount int `json:"env_count"` + ArtifactCount int `json:"artifact_count"` +} + +func GetOverviewStat(log *zap.SugaredLogger) (*Overview, error) { + overview := new(Overview) + projectCount, err := templaterepo.NewProductColl().Count() + if err != nil { + log.Errorf("Failed to get project count err:%s", err) + return nil, err + } + overview.ProjectCount = int(projectCount) + + clusterCount, err := commonrepo.NewK8SClusterColl().Count() + if err != nil { + log.Errorf("Failed to get cluster count err:%s", err) + return nil, err + } + overview.ClusterCount = int(clusterCount) + + workflowCount, err := commonrepo.NewWorkflowColl().Count() + if err != nil { + log.Errorf("Failed to get workflow count err:%s", err) + return nil, err + } + overview.WorkflowCount = int(workflowCount) + + envCount, err := commonrepo.NewProductColl().EnvCount() + if err != nil { + log.Errorf("Failed to get env count err:%s", err) + return nil, err + } + overview.EnvCount = int(envCount) + + _, artifactCount, err := commonrepo.NewDeliveryArtifactColl().List(&commonrepo.DeliveryArtifactArgs{OnlyCount: true}) + if err != nil { + log.Errorf("Failed to get artifact count err:%s", err) + return nil, err + } + overview.ArtifactCount = artifactCount + + serviceCount, err := commonrepo.NewServiceColl().Count("") + if err != nil { + log.Errorf("Failed to get service count err:%s", err) + return nil, err + } + overview.ServiceCount = int(serviceCount) + + return overview, nil +} diff --git a/pkg/microservice/aslan/core/stat/service/test.go b/pkg/microservice/aslan/core/stat/service/test.go new file mode 100644 index 0000000000000000000000000000000000000000..e759fd8516053a4c4cbffa61c857f8566e27f2db --- /dev/null +++ b/pkg/microservice/aslan/core/stat/service/test.go @@ -0,0 +1,81 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" +) + +type testDashboard struct { + TotalCaseCount int `json:"total_case_count"` + TotalExecCount int `json:"total_exec_count"` + Success int `json:"success"` + AverageDuration int64 `json:"average_duration"` +} + +func GetTestDashboard(log *zap.SugaredLogger) (*testDashboard, error) { + var ( + testDashboard = new(testDashboard) + existTestTasks = make([]*models.TestTaskStat, 0) + totalCaseCount = 0 + totalSuccess = 0 + totalFailure = 0 + totalDuration int64 + ) + + tests, err := commonrepo.NewTestingColl().List(&commonrepo.ListTestOption{}) + if err != nil { + log.Errorf("Failed to list testing err:%s", err) + return nil, err + } + + testNames := sets.String{} + for _, test := range tests { + testNames.Insert(test.Name) + } + testTasks, err := commonrepo.NewTestTaskStatColl().ListTestTask() + if err != nil { + log.Errorf("Failed to list TestTaskStat err:%s", err) + return nil, err + } + for _, testTask := range testTasks { + if testNames.Has(testTask.Name) { + existTestTasks = append(existTestTasks, testTask) + } + } + + for _, existTestTask := range existTestTasks { + totalCaseCount += existTestTask.TestCaseNum + totalSuccess += existTestTask.TotalSuccess + totalFailure += existTestTask.TotalFailure + totalDuration += existTestTask.TotalDuration + } + testDashboard.TotalCaseCount = totalCaseCount + testDashboard.TotalExecCount = totalSuccess + totalFailure + testDashboard.Success = totalSuccess + if testDashboard.TotalExecCount == 0 { + testDashboard.AverageDuration = 0 + return testDashboard, nil + } + testDashboard.AverageDuration = totalDuration / int64(testDashboard.TotalExecCount) + + return testDashboard, nil +} diff --git a/pkg/microservice/aslan/core/stat/service/test_stat.go b/pkg/microservice/aslan/core/stat/service/test_stat.go new file mode 100644 index 0000000000000000000000000000000000000000..3495f2ab110c3f9de2b0ad117986aff28899ebe8 --- /dev/null +++ b/pkg/microservice/aslan/core/stat/service/test_stat.go @@ -0,0 +1,528 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "encoding/xml" + "fmt" + "io/ioutil" + "math" + "os" + "sort" + "strings" + "time" + + "github.com/jinzhu/now" + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + taskmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" + commonmongodb "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" + s3service "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3" + "github.com/koderover/zadig/pkg/microservice/aslan/core/stat/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/stat/repository/mongodb" + "github.com/koderover/zadig/pkg/setting" + s3tool "github.com/koderover/zadig/pkg/tool/s3" + "github.com/koderover/zadig/pkg/util" +) + +func InitTestStat(log *zap.SugaredLogger) error { + option := &commonmongodb.ListAllTaskOption{Type: config.WorkflowType} + count, err := mongodb.NewTestStatColl().FindCount() + if err != nil { + log.Errorf("testStat FindCount err:%v", err) + return fmt.Errorf("testStat FindCount err:%v", err) + } + if count > 0 { + option.CreateTime = time.Now().AddDate(0, 0, -1).Unix() + } + //获取所有的项目名称 + allProducts, err := templaterepo.NewProductColl().List() + if err != nil { + log.Errorf("testStat ProductTmpl List err:%v", err) + return fmt.Errorf("testStat ProductTmpl List err:%v", err) + } + for _, product := range allProducts { + option.ProductNames = []string{product.ProductName} + allTasks, err := commonmongodb.NewTaskColl().ListAllTasks(option) + if err != nil { + log.Errorf("pipeline list err:%v", err) + return fmt.Errorf("pipeline list err:%v", err) + } + + taskDateMap := make(map[string][]*taskmodels.Task) + if len(allTasks) > 0 { + //将task的时间戳转成日期，以日期为单位分组 + for _, task := range allTasks { + time := time.Unix(task.CreateTime, 0) + date := time.Format(config.Date) + taskDateMap[date] = append(taskDateMap[date], task) + } + } else { + time := time.Now().AddDate(0, 0, -1) + date := time.Format(config.Date) + taskDateMap[date] = []*taskmodels.Task{ + {Stages: []*commonmodels.Stage{}}, + } + } + + taskDateKeys := make([]string, 0, len(taskDateMap)) + for taskDateMapKey := range taskDateMap { + taskDateKeys = append(taskDateKeys, taskDateMapKey) + } + sort.Strings(taskDateKeys) + + defaultS3Storage, _ := s3service.FindDefaultS3() + + for _, taskDate := range taskDateKeys { + var ( + totalSuccess = 0 + totalFailure = 0 + totalTimeout = 0 + totalDuration int64 + totalDeployCount = 0 + totalTestCount = 0 + totalTestCase = 0 + ) + //循环task任务获取需要的数据 + for _, taskPreview := range taskDateMap[taskDate] { + stages := taskPreview.Stages + for _, subStage := range stages { + taskType := subStage.TaskType + switch taskType { + case config.TaskTestingV2: + // 获取构建时长 + for _, subTask := range subStage.SubTasks { + testInfo, err := base.ToTestingTask(subTask) + if err != nil { + log.Errorf("TestStat ToTestingTask err:%v", err) + continue + } + if testInfo.TaskStatus == config.StatusPassed { + totalSuccess++ + } else if testInfo.TaskStatus == config.StatusFailed { + totalFailure++ + } else if testInfo.TaskStatus == config.StatusTimeout { + totalTimeout++ + } else { + continue + } + + totalDuration += testInfo.EndTime - testInfo.StartTime + totalTestCount++ + + if testInfo.JobCtx.TestType != setting.PerformanceTest { + func() { + testJobName := strings.Replace(strings.ToLower(fmt.Sprintf("%s-%s-%d-%s-%s", + config.WorkflowType, taskPreview.PipelineName, taskPreview.TaskID, config.TaskTestingV2, testInfo.TestModuleName)), "_", "-", -1) + + filename, err := util.GenerateTmpFile() + defer func() { + _ = os.Remove(filename) + }() + + if err != nil { + log.Errorf("failed to genarate tmp file, err: %s", err) + return + } + + objectKey := defaultS3Storage.GetObjectPath(fmt.Sprintf("%s/%d/%s/%s", taskPreview.PipelineName, taskPreview.TaskID, "test", testJobName)) + forcedPathStyle := true + if defaultS3Storage.Provider == setting.ProviderSourceAli { + forcedPathStyle = false + } + client, err := s3tool.NewClient(defaultS3Storage.Endpoint, defaultS3Storage.Ak, defaultS3Storage.Sk, defaultS3Storage.Insecure, forcedPathStyle) + if err != nil { + log.Errorf("Failed to create s3 client for download, error: %s", err) + return + } + if err = client.Download(defaultS3Storage.Bucket, objectKey, filename); err != nil { + log.Errorf("s3Service download err:%s", err) + return + } + + b, err := ioutil.ReadFile(filename) + if err != nil { + log.Error(fmt.Sprintf("InitTestStat get local test result file error: %v", err)) + return + } + testReport := new(commonmodels.TestReport) + if err := xml.Unmarshal(b, &testReport.FunctionTestSuite); err != nil { + log.Error(fmt.Sprintf("unmarshal it report xml error: %v", err)) + return + } + totalTestCase += testReport.FunctionTestSuite.Tests + }() + } + } + case config.TaskDeploy: + totalDeployCount++ + } + } + } + + testStat := new(models.TestStat) + testStat.ProductName = product.ProductName + testStat.TotalSuccess = totalSuccess + testStat.TotalFailure = totalFailure + testStat.TotalTimeout = totalTimeout + testStat.TotalDuration = totalDuration + testStat.TotalTestCount = totalTestCount + testStat.TotalDeployCount = totalDeployCount + testStat.TotalTestCase = totalTestCase + testStat.Date = taskDate + tt, _ := time.ParseInLocation(config.Date, taskDate, time.Local) + testStat.CreateTime = tt.Unix() + testStat.UpdateTime = time.Now().Unix() + + err := mongodb.NewTestStatColl().Create(testStat) + if err != nil { //插入失败就更新 + err = mongodb.NewTestStatColl().Update(testStat) + if err != nil { + log.Errorf("TestStat Update err:%v", err) + continue + } + } + } + } + return nil +} + +type testStatDailyArg struct { + Date string `json:"date"` + AverageDuration int `json:"averageDuration"` +} + +func GetTestAverageMeasure(startDate, endDate int64, productNames []string, log *zap.SugaredLogger) ([]*testStatDailyArg, error) { + testStats, err := mongodb.NewTestStatColl().ListTestStat(&mongodb.TestStatOption{StartDate: startDate, EndDate: endDate, IsAsc: true, ProductNames: productNames}) + if err != nil { + log.Errorf("ListTestStat err:%v", err) + return nil, fmt.Errorf("ListTestStat err:%v", err) + } + testStatMap := make(map[string][]*models.TestStat) + for _, testStat := range testStats { + if _, isExist := testStatMap[testStat.Date]; isExist { + testStatMap[testStat.Date] = append(testStatMap[testStat.Date], testStat) + } else { + tempTestStats := make([]*models.TestStat, 0) + tempTestStats = append(tempTestStats, testStat) + testStatMap[testStat.Date] = tempTestStats + } + } + testStatDateKeys := make([]string, 0, len(testStatMap)) + for testStatDateMapKey := range testStatMap { + testStatDateKeys = append(testStatDateKeys, testStatDateMapKey) + } + sort.Strings(testStatDateKeys) + testStatDailyArgs := make([]*testStatDailyArg, 0) + for _, testStatDateKey := range testStatDateKeys { + totalDuration := 0 + totalTestCount := 0 + for _, testStat := range testStatMap[testStatDateKey] { + totalDuration += int(testStat.TotalDuration) + totalTestCount += testStat.TotalTestCount + } + testStatDailyArg := new(testStatDailyArg) + testStatDailyArg.Date = testStatDateKey + if totalTestCount > 0 { + testStatDailyArg.AverageDuration = int(math.Floor(float64(totalDuration)/float64(totalTestCount) + 0.5)) + } + + testStatDailyArgs = append(testStatDailyArgs, testStatDailyArg) + } + return testStatDailyArgs, nil +} + +type testCaseStat struct { + Day int64 `json:"day"` + TestCase int `json:"testCase"` +} + +func GetTestCaseMeasure(startDate, endDate int64, productNames []string, log *zap.SugaredLogger) ([]*testCaseStat, error) { + testStats, err := mongodb.NewTestStatColl().ListTestStat(&mongodb.TestStatOption{StartDate: startDate, EndDate: endDate, IsAsc: true, ProductNames: productNames}) + if err != nil { + log.Errorf("ListTestStat err:%v", err) + return nil, fmt.Errorf("ListTestStat err:%v", err) + } + testStatMap := make(map[string][]*models.TestStat) + for _, testStat := range testStats { + if _, isExist := testStatMap[testStat.Date]; isExist { + testStatMap[testStat.Date] = append(testStatMap[testStat.Date], testStat) + } else { + tempTestStats := make([]*models.TestStat, 0) + tempTestStats = append(tempTestStats, testStat) + testStatMap[testStat.Date] = tempTestStats + } + } + testStatDateKeys := make([]string, 0, len(testStatMap)) + for testStatDateMapKey := range testStatMap { + testStatDateKeys = append(testStatDateKeys, testStatDateMapKey) + } + sort.Sort(sort.Reverse(sort.StringSlice(testStatDateKeys))) + testCaseStats := make([]*testCaseStat, 0) + totalTestCase := 0 + if len(testStatMap) <= config.Day { + for _, testStat := range testStats { + totalTestCase += testStat.TotalTestCount * testStat.TotalTestCase + } + testCaseStat := &testCaseStat{ + Day: time.Now().Unix(), + TestCase: totalTestCase, + } + testCaseStats = append(testCaseStats, testCaseStat) + } else { + for index, testStatDate := range testStatDateKeys { + for _, testStat := range testStatMap[testStatDate] { + totalTestCase += testStat.TotalTestCount * testStat.TotalTestCase + } + + if ((index + 1) % config.Day) == 0 { + day, err := time.ParseInLocation("2006-01-02", testStatDate, time.Local) + if err != nil { + log.Errorf("Failed to parse %s as time, err:", testStatDate, err) + return nil, err + } + testCaseStat := &testCaseStat{ + Day: day.Unix(), + TestCase: totalTestCase, + } + testCaseStats = append(testCaseStats, testCaseStat) + totalTestCase = 0 + } + } + } + + return testCaseStats, nil +} + +type testDeployStat struct { + Day int64 `json:"day"` + DeliveryDeploy int `json:"deliveryDeploy"` +} + +func GetTestDeliveryDeployMeasure(startDate, endDate int64, productNames []string, log *zap.SugaredLogger) ([]*testDeployStat, error) { + testStats, err := mongodb.NewTestStatColl().ListTestStat(&mongodb.TestStatOption{StartDate: startDate, EndDate: endDate, IsAsc: true, ProductNames: productNames}) + if err != nil { + log.Errorf("ListTestStat err:%v", err) + return nil, fmt.Errorf("ListTestStat err:%v", err) + } + testStatMap := make(map[string][]*models.TestStat) + for _, testStat := range testStats { + if _, isExist := testStatMap[testStat.Date]; isExist { + testStatMap[testStat.Date] = append(testStatMap[testStat.Date], testStat) + } else { + tempTestStats := make([]*models.TestStat, 0) + tempTestStats = append(tempTestStats, testStat) + testStatMap[testStat.Date] = tempTestStats + } + } + testStatDateKeys := make([]string, 0, len(testStatMap)) + for testStatDateMapKey := range testStatMap { + testStatDateKeys = append(testStatDateKeys, testStatDateMapKey) + } + sort.Sort(sort.Reverse(sort.StringSlice(testStatDateKeys))) + testDeployStats := make([]*testDeployStat, 0) + totalDeploy := 0 + if len(testStatMap) <= config.Day { + for _, testStat := range testStats { + totalDeploy += testStat.TotalDeployCount + } + testDeployStat := &testDeployStat{ + Day: time.Now().Unix(), + DeliveryDeploy: totalDeploy, + } + testDeployStats = append(testDeployStats, testDeployStat) + } else { + for index, testStatDate := range testStatDateKeys { + for _, testStat := range testStatMap[testStatDate] { + totalDeploy += testStat.TotalDeployCount + } + + if ((index + 1) % config.Day) == 0 { + day, err := time.ParseInLocation("2006-01-02", testStatDate, time.Local) + if err != nil { + log.Errorf("Failed to parse %s as time, err:", testStatDate, err) + return nil, err + } + testDeployStat := &testDeployStat{ + Day: day.Unix(), + DeliveryDeploy: totalDeploy, + } + testDeployStats = append(testDeployStats, testDeployStat) + totalDeploy = 0 + } + } + } + + return testDeployStats, nil +} + +type testStatTotal struct { + TotalSuccess int `json:"totalSuccess"` + TotalFailure int `json:"totalFailure"` +} + +func GetTestHealthMeasure(startDate, endDate int64, productNames []string, log *zap.SugaredLogger) (*testStatTotal, error) { + testStats, err := mongodb.NewTestStatColl().ListTestStat(&mongodb.TestStatOption{StartDate: startDate, EndDate: endDate, IsAsc: true, ProductNames: productNames}) + if err != nil { + log.Errorf("ListTestStat err:%v", err) + return nil, fmt.Errorf("ListTestStat err:%v", err) + } + var ( + totalSuccess = 0 + totalFailure = 0 + ) + for _, testStat := range testStats { + totalSuccess += testStat.TotalSuccess + totalFailure += testStat.TotalFailure + } + testStatTotal := &testStatTotal{ + TotalSuccess: totalSuccess, + TotalFailure: totalFailure, + } + return testStatTotal, nil +} + +type testTrend struct { + *CurrentDay + Sum []*sumData `json:"sum"` +} + +func GetTestTrendMeasure(startDate, endDate int64, productNames []string, log *zap.SugaredLogger) (*testTrend, error) { + todayDate := now.BeginningOfDay().Unix() + var ( + totalSuccess = 0 + totalFailure = 0 + totalTimeout = 0 + ) + allTasks, err := commonmongodb.NewTaskColl().ListAllTasks(&commonmongodb.ListAllTaskOption{Type: config.WorkflowType, CreateTime: todayDate, ProductNames: productNames}) + if err != nil { + log.Errorf("ListAllTasks err:%v", err) + return nil, fmt.Errorf("ListAllTasks err:%v", err) + } + for _, taskPreview := range allTasks { + stages := taskPreview.Stages + for _, subStage := range stages { + taskType := subStage.TaskType + switch taskType { + case config.TaskTestingV2: + // 获取构建时长 + for _, subTask := range subStage.SubTasks { + testInfo, err := base.ToTestingTask(subTask) + if err != nil { + log.Errorf("TestStat ToTestingTask err:%v", err) + continue + } + if testInfo.TaskStatus == config.StatusPassed { + totalSuccess++ + } else if testInfo.TaskStatus == config.StatusFailed { + totalFailure++ + } else if testInfo.TaskStatus == config.StatusTimeout { + totalTimeout++ + } else { + continue + } + } + } + } + } + currDay := &CurrentDay{ + Success: totalSuccess, + Failure: totalFailure, + Timeout: totalTimeout, + } + sumDatas := make([]*sumData, 0) + + testStats, err := mongodb.NewTestStatColl().ListTestStat(&mongodb.TestStatOption{StartDate: startDate, EndDate: endDate, IsAsc: false, ProductNames: productNames}) + if err != nil { + log.Errorf("ListTestStat err:%v", err) + return nil, fmt.Errorf("ListTestStat err:%v", err) + } + testStatMap := make(map[string][]*models.TestStat) + for _, testStat := range testStats { + if _, isExist := testStatMap[testStat.Date]; isExist { + testStatMap[testStat.Date] = append(testStatMap[testStat.Date], testStat) + } else { + tempTestStats := make([]*models.TestStat, 0) + tempTestStats = append(tempTestStats, testStat) + testStatMap[testStat.Date] = tempTestStats + } + } + testStatDateKeys := make([]string, 0, len(testStatMap)) + for testStatDateMapKey := range testStatMap { + testStatDateKeys = append(testStatDateKeys, testStatDateMapKey) + } + sort.Sort(sort.Reverse(sort.StringSlice(testStatDateKeys))) + + totalSuccess = 0 + totalFailure = 0 + totalTimeout = 0 + if len(testStatMap) <= config.Day { + for _, testStat := range testStats { + totalSuccess += testStat.TotalSuccess + totalFailure += testStat.TotalFailure + totalTimeout += testStat.TotalTimeout + } + sumData := &sumData{ + Day: time.Now().Unix(), + CurrentDay: &CurrentDay{ + Success: totalSuccess, + Failure: totalFailure, + Timeout: totalTimeout, + }, + } + sumDatas = append(sumDatas, sumData) + } else { + for index, testStatDate := range testStatDateKeys { + for _, testStat := range testStatMap[testStatDate] { + totalSuccess += testStat.TotalSuccess + totalFailure += testStat.TotalFailure + totalTimeout += testStat.TotalTimeout + } + + if ((index + 1) % config.Day) == 0 { + day, err := time.ParseInLocation("2006-01-02", testStatDate, time.Local) + if err != nil { + log.Errorf("Failed to parse %s as time, err:", testStatDate, err) + return nil, err + } + sumData := &sumData{ + Day: day.Unix(), + CurrentDay: &CurrentDay{ + Success: totalSuccess, + Failure: totalFailure, + Timeout: totalTimeout, + }, + } + sumDatas = append(sumDatas, sumData) + totalSuccess = 0 + totalFailure = 0 + totalTimeout = 0 + } + } + } + + testTrend := &testTrend{ + CurrentDay: currDay, + Sum: sumDatas, + } + + return testTrend, nil +} diff --git a/pkg/microservice/aslan/core/system/handler/announcement.go b/pkg/microservice/aslan/core/system/handler/announcement.go index 739b84cab8f5ca0a2e4a072ec2354bc78340f91c..30227aa79fd4b9aac209eb23c512b7a7583d1c79 100644 --- a/pkg/microservice/aslan/core/system/handler/announcement.go +++ b/pkg/microservice/aslan/core/system/handler/announcement.go @@ -35,7 +35,7 @@ func CreateAnnouncement(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid notify args") return } - ctx.Err = service.CreateAnnouncement(ctx.Username, args, ctx.Logger) + ctx.Err = service.CreateAnnouncement(ctx.UserName, args, ctx.Logger) } func UpdateAnnouncement(c *gin.Context) { @@ -48,21 +48,21 @@ func UpdateAnnouncement(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid notify args") return } - ctx.Err = service.UpdateAnnouncement(ctx.Username, args.ID.Hex(), args, ctx.Logger) + ctx.Err = service.UpdateAnnouncement(ctx.UserName, args.ID.Hex(), args, ctx.Logger) } func PullAllAnnouncement(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = service.PullAllAnnouncement(ctx.Username, ctx.Logger) + ctx.Resp, ctx.Err = service.PullAllAnnouncement(ctx.UserName, ctx.Logger) } func PullNotifyAnnouncement(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = service.PullNotifyAnnouncement(ctx.Username, ctx.Logger) + ctx.Resp, ctx.Err = service.PullNotifyAnnouncement(ctx.UserName, ctx.Logger) } func DeleteAnnouncement(c *gin.Context) { @@ -71,5 +71,5 @@ func DeleteAnnouncement(c *gin.Context) { ID := c.Param("id") - ctx.Err = service.DeleteAnnouncement(ctx.Username, ID, ctx.Logger) + ctx.Err = service.DeleteAnnouncement(ctx.UserName, ID, ctx.Logger) } diff --git a/pkg/microservice/aslan/core/system/handler/basic_images.go b/pkg/microservice/aslan/core/system/handler/basic_images.go index 5e4a7ebfe199e25d0d3a34bd0a2af2456bd00d8b..157e668d085a3b4fa98b661837dca7b91e862a86 100644 --- a/pkg/microservice/aslan/core/system/handler/basic_images.go +++ b/pkg/microservice/aslan/core/system/handler/basic_images.go @@ -43,7 +43,7 @@ func ListBasicImages(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = service.ListBasicImages(c.Query("image_from"), ctx.Logger) + ctx.Resp, ctx.Err = service.ListBasicImages(c.Query("image_from"), c.Query("image_type"), ctx.Logger) } func CreateBasicImage(c *gin.Context) { @@ -58,7 +58,7 @@ func CreateBasicImage(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("CreateBasicImage json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, "", "新增", "基础镜像", fmt.Sprintf("label:%s", args.Label), string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, "", "新增", "基础镜像", fmt.Sprintf("label:%s", args.Label), string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) @@ -66,7 +66,7 @@ func CreateBasicImage(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid BasicImage args") return } - args.UpdateBy = ctx.Username + args.UpdateBy = ctx.UserName ctx.Err = service.CreateBasicImage(args, ctx.Logger) } @@ -83,7 +83,7 @@ func UpdateBasicImage(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("UpdateBasicImage json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, "", "更新", "基础镜像", fmt.Sprintf("id:%s", args.ID), string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, "", "更新", "基础镜像", fmt.Sprintf("id:%s", args.ID), string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) @@ -91,7 +91,7 @@ func UpdateBasicImage(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid BasicImage args") return } - args.UpdateBy = ctx.Username + args.UpdateBy = ctx.UserName ctx.Err = service.UpdateBasicImage(c.Param("id"), args, ctx.Logger) } @@ -100,7 +100,7 @@ func DeleteBasicImage(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - internalhandler.InsertOperationLog(c, ctx.Username, "", "删除", "基础镜像", fmt.Sprintf("id:%s", c.Param("id")), "", ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, "", "删除", "基础镜像", fmt.Sprintf("id:%s", c.Param("id")), "", ctx.Logger) ctx.Err = service.DeleteBasicImage(c.Param("id"), ctx.Logger) } diff --git a/pkg/microservice/aslan/core/system/handler/concurrency.go b/pkg/microservice/aslan/core/system/handler/concurrency.go new file mode 100644 index 0000000000000000000000000000000000000000..b6c9d4b14741ac6ef4ae72fbcd8aa9eb7a76c812 --- /dev/null +++ b/pkg/microservice/aslan/core/system/handler/concurrency.go @@ -0,0 +1,52 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "errors" + + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/system/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" +) + +func GetWorkflowConcurrency(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.GetWorkflowConcurrency() +} + +func UpdateWorkflowConcurrency(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(service.WorkflowConcurrencySettings) + if err := c.BindJSON(args); err != nil { + ctx.Err = err + return + } + + // args validation + if args.BuildConcurrency <= 0 || args.WorkflowConcurrency <= 0 { + ctx.Err = errors.New("concurrency cannot be less than 1") + return + } + + ctx.Err = service.UpdateWorkflowConcurrency(args.WorkflowConcurrency, args.BuildConcurrency, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/system/handler/external_link.go b/pkg/microservice/aslan/core/system/handler/external_link.go new file mode 100644 index 0000000000000000000000000000000000000000..bf20968ec1a8d57fd97b031a883805dd1da9d25e --- /dev/null +++ b/pkg/microservice/aslan/core/system/handler/external_link.go @@ -0,0 +1,97 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + + "github.com/gin-gonic/gin" + + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/system/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/log" +) + +func ListExternalLinks(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.ListExternalLinks(ctx.Logger) +} + +func CreateExternalLink(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(commonmodels.ExternalLink) + data, err := c.GetRawData() + if err != nil { + log.Errorf("CreateExternalLink c.GetRawData() err : %s", err) + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("CreateExternalLink json.Unmarshal err : %s", err) + } + internalhandler.InsertOperationLog(c, ctx.UserName, "", "新增", "系统配置-快捷链接", fmt.Sprintf("name:%s url:%s", args.Name, args.URL), string(data), ctx.Logger) + + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + + if err := c.ShouldBindJSON(&args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid externalLink args") + return + } + args.UpdateBy = ctx.UserName + + ctx.Err = service.CreateExternalLink(args, ctx.Logger) +} + +func UpdateExternalLink(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(commonmodels.ExternalLink) + data, err := c.GetRawData() + if err != nil { + log.Errorf("UpdateExternal c.GetRawData() err : %s", err) + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("UpdateExternal json.Unmarshal err : %s", err) + } + internalhandler.InsertOperationLog(c, ctx.UserName, "", "更新", "系统配置-快捷链接", fmt.Sprintf("name:%s url:%s", args.Name, args.URL), string(data), ctx.Logger) + + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + + if err := c.ShouldBindJSON(&args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid externalLink args") + return + } + args.UpdateBy = ctx.UserName + + ctx.Err = service.UpdateExternalLink(c.Param("id"), args, ctx.Logger) +} + +func DeleteExternalLink(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + internalhandler.InsertOperationLog(c, ctx.UserName, "", "删除", "系统配置-快捷链接", fmt.Sprintf("id:%s", c.Param("id")), "", ctx.Logger) + ctx.Err = service.DeleteExternalLink(c.Param("id"), ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/system/handler/external_system.go b/pkg/microservice/aslan/core/system/handler/external_system.go new file mode 100644 index 0000000000000000000000000000000000000000..97860c7ff68faee4b66b381ba492cfe1b7a6d181 --- /dev/null +++ b/pkg/microservice/aslan/core/system/handler/external_system.go @@ -0,0 +1,133 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/system/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/log" +) + +func CreateExternalSystem(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(service.ExternalSystemDetail) + data, err := c.GetRawData() + if err != nil { + log.Errorf("CreateExternalSystem GetRawData err : %s", err) + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("CreateExternalSystem Unmarshal err : %s", err) + } + internalhandler.InsertOperationLog(c, ctx.UserName, "", "新增", "系统配置-外部系统", fmt.Sprintf("name:%s server:%s", args.Name, args.Server), string(data), ctx.Logger) + + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + + if args.Name == "" || args.Server == "" { + ctx.Err = errors.New("name and server must be provided") + return + } + + ctx.Err = service.CreateExternalSystem(args, ctx.Logger) +} + +type listQuery struct { + PageSize int64 `json:"page_size" form:"page_size,default=100"` + PageNum int64 `json:"page_num" form:"page_num,default=1"` +} + +type listExternalResp struct { + SystemList []*service.ExternalSystemDetail `json:"external_system"` + Total int64 `json:"total"` +} + +func ListExternalSystem(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + // Query Verification + args := &listQuery{} + if err := c.ShouldBindQuery(args); err != nil { + ctx.Err = err + return + } + + encryptedKey := c.Query("encryptedKey") + if len(encryptedKey) == 0 { + ctx.Err = e.ErrInvalidParam + return + } + systemList, length, err := service.ListExternalSystem(encryptedKey, args.PageNum, args.PageSize, ctx.Logger) + if err == nil { + ctx.Resp = &listExternalResp{ + SystemList: systemList, + Total: length, + } + return + } + ctx.Err = err +} + +func GetExternalSystemDetail(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.GetExternalSystemDetail(c.Param("id"), ctx.Logger) +} + +func UpdateExternalSystem(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + req := new(service.ExternalSystemDetail) + data, err := c.GetRawData() + if err != nil { + log.Errorf("UpdateExternalSystem GetRawData err : %s", err) + } + if err = json.Unmarshal(data, req); err != nil { + log.Errorf("UpdateExternalSystem Unmarshal err : %s", err) + } + internalhandler.InsertOperationLog(c, ctx.UserName, "", "更新", "系统配置-外部系统", fmt.Sprintf("name:%s server:%s", req.Name, req.Server), string(data), ctx.Logger) + + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + + if req.Name == "" || req.Server == "" { + ctx.Err = errors.New("name and server must be provided") + return + } + + ctx.Err = service.UpdateExternalSystem(c.Param("id"), req, ctx.Logger) +} + +func DeleteExternalSystem(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + internalhandler.InsertOperationLog(c, ctx.UserName, "", "删除", "系统配置-外部系统", fmt.Sprintf("id:%s", c.Param("id")), "", ctx.Logger) + + ctx.Err = service.DeleteExternalSystem(c.Param("id"), ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/system/handler/helm.go b/pkg/microservice/aslan/core/system/handler/helm.go index 928791483835d5dd77e56d1590f0383e36bed8f5..4f5b4f0b0cd06562af7159b4e92225df9a6704a1 100644 --- a/pkg/microservice/aslan/core/system/handler/helm.go +++ b/pkg/microservice/aslan/core/system/handler/helm.go @@ -31,8 +31,12 @@ import ( func ListHelmRepos(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - - ctx.Resp, ctx.Err = commonservice.ListHelmRepos(ctx.Logger) + encryptedKey := c.Query("encryptedKey") + if len(encryptedKey) == 0 { + ctx.Err = e.ErrInvalidParam + return + } + ctx.Resp, ctx.Err = commonservice.ListHelmRepos(encryptedKey, ctx.Logger) } func CreateHelmRepo(c *gin.Context) { @@ -44,11 +48,12 @@ func CreateHelmRepo(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid helmRepo json args") return } - args.UpdateBy = ctx.Username + args.UpdateBy = ctx.UserName if _, err := url.Parse(args.URL); err != nil { ctx.Err = e.ErrInvalidParam.AddDesc("invalid url") return } + ctx.Err = service.CreateHelmRepo(args, ctx.Logger) } @@ -61,7 +66,7 @@ func UpdateHelmRepo(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid helmRepo json args") return } - args.UpdateBy = ctx.Username + args.UpdateBy = ctx.UserName ctx.Err = service.UpdateHelmRepo(c.Param("id"), args, ctx.Logger) } @@ -71,3 +76,10 @@ func DeleteHelmRepo(c *gin.Context) { ctx.Err = service.DeleteHelmRepo(c.Param("id"), ctx.Logger) } + +func ListCharts(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.ListCharts(c.Param("name"), ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/system/handler/install.go b/pkg/microservice/aslan/core/system/handler/install.go index f2347d9fe3e82b4162963ac7794447e5a085640a..35532d16249d3a96932f00c12c38a9d84d4998c2 100644 --- a/pkg/microservice/aslan/core/system/handler/install.go +++ b/pkg/microservice/aslan/core/system/handler/install.go @@ -44,14 +44,14 @@ func CreateInstall(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("CreateInstall json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, "", "新增", "系统设置-应用设置", fmt.Sprintf("应用名称:%s,应用版本:%s", args.Name, args.Version), string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, "", "新增", "系统设置-应用设置", fmt.Sprintf("应用名称:%s,应用版本:%s", args.Name, args.Version), string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) if err := c.ShouldBindWith(&args, binding.JSON); err != nil { ctx.Err = e.ErrInvalidParam.AddDesc("invalid Install args") return } - args.UpdateBy = ctx.Username + args.UpdateBy = ctx.UserName ctx.Err = service.CreateInstall(args, ctx.Logger) } @@ -68,14 +68,14 @@ func UpdateInstall(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("UpdateInstall json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, "", "更新", "系统设置-应用设置", fmt.Sprintf("应用名称:%s,应用版本:%s", args.Name, args.Version), string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, "", "更新", "系统设置-应用设置", fmt.Sprintf("应用名称:%s,应用版本:%s", args.Name, args.Version), string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) if err := c.ShouldBindWith(&args, binding.JSON); err != nil { ctx.Err = e.ErrInvalidParam.AddDesc("invalid Install args") return } - args.UpdateBy = ctx.Username + args.UpdateBy = ctx.UserName name := args.Name version := args.Version if name == "" || version == "" { @@ -116,7 +116,7 @@ func DeleteInstall(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("DeleteInstall json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, "", "删除", "系统设置-应用设置", fmt.Sprintf("应用名称:%s,应用版本:%s", args.Name, args.Version), string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, "", "删除", "系统设置-应用设置", fmt.Sprintf("应用名称:%s,应用版本:%s", args.Name, args.Version), string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) if err := c.ShouldBindWith(&args, binding.JSON); err != nil { diff --git a/pkg/microservice/aslan/core/system/handler/jenkins.go b/pkg/microservice/aslan/core/system/handler/jenkins.go index 20e7962191625051361fa4b1a690c10c92af0fab..e333673d9067835d3a94b260f12a4daeb2bc282c 100644 --- a/pkg/microservice/aslan/core/system/handler/jenkins.go +++ b/pkg/microservice/aslan/core/system/handler/jenkins.go @@ -27,6 +27,22 @@ import ( e "github.com/koderover/zadig/pkg/tool/errors" ) +type CheckJenkinsIntegrationResp struct { + Exists bool `json:"exists"` +} + +func CheckJenkinsIntegration(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + resp, err := service.ListJenkinsIntegration("", ctx.Logger) + if err != nil || len(resp) == 0 { + ctx.Resp = &CheckJenkinsIntegrationResp{Exists: false} + return + } + ctx.Resp = &CheckJenkinsIntegrationResp{Exists: true} +} + func CreateJenkinsIntegration(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() @@ -36,7 +52,7 @@ func CreateJenkinsIntegration(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid jenkinsIntegration json args") return } - args.UpdateBy = ctx.Username + args.UpdateBy = ctx.UserName if _, err := url.Parse(args.URL); err != nil { ctx.Err = e.ErrInvalidParam.AddDesc("invalid url") return @@ -48,7 +64,12 @@ func ListJenkinsIntegration(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = service.ListJenkinsIntegration(ctx.Logger) + encryptedKey := c.Query("encryptedKey") + if len(encryptedKey) == 0 { + ctx.Err = e.ErrInvalidParam + return + } + ctx.Resp, ctx.Err = service.ListJenkinsIntegration(encryptedKey, ctx.Logger) } func UpdateJenkinsIntegration(c *gin.Context) { @@ -60,7 +81,7 @@ func UpdateJenkinsIntegration(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid jenkinsIntegration json args") return } - args.UpdateBy = ctx.Username + args.UpdateBy = ctx.UserName ctx.Err = service.UpdateJenkinsIntegration(c.Param("id"), args, ctx.Logger) } @@ -88,12 +109,12 @@ func ListJobNames(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = service.ListJobNames(ctx.Logger) + ctx.Resp, ctx.Err = service.ListJobNames(c.Param("id"), ctx.Logger) } func ListJobBuildArgs(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = service.ListJobBuildArgs(c.Param("jobName"), ctx.Logger) + ctx.Resp, ctx.Err = service.ListJobBuildArgs(c.Param("id"), c.Param("jobName"), ctx.Logger) } diff --git a/pkg/microservice/aslan/core/system/handler/login.go b/pkg/microservice/aslan/core/system/handler/login.go new file mode 100644 index 0000000000000000000000000000000000000000..4925e6abaccecf6fdfe7ce2a552740785593dd23 --- /dev/null +++ b/pkg/microservice/aslan/core/system/handler/login.go @@ -0,0 +1,44 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/system/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" +) + +func GetDefaultLogin(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.GetDefaultLogin(ctx.Logger) +} + +func UpdateDefaultLogin(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(service.UpdateDefaultLoginParams) + if err := c.BindJSON(args); err != nil { + ctx.Err = err + return + } + + ctx.Err = service.UpdateDefaultLogin(args.DefaultLogin, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/system/handler/notify.go b/pkg/microservice/aslan/core/system/handler/notify.go index 5093cea28bc736eb4ee7cfe66961f2b22c7721bf..f654e8a195416d9f10b3b9c84e97dd79f8805ed1 100644 --- a/pkg/microservice/aslan/core/system/handler/notify.go +++ b/pkg/microservice/aslan/core/system/handler/notify.go @@ -31,7 +31,7 @@ func PullNotify(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = service.PullNotify(ctx.Username, ctx.Logger) + ctx.Resp, ctx.Err = service.PullNotify(ctx.UserName, ctx.Logger) } type readNotificationsArgs struct { @@ -48,7 +48,7 @@ func ReadNotify(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid readnotificationsargs") return } - ctx.Err = service.ReadNotify(ctx.Username, args.IDs, ctx.Logger) + ctx.Err = service.ReadNotify(ctx.UserName, args.IDs, ctx.Logger) } type deleteNotificationsArgs struct { @@ -65,7 +65,7 @@ func DeleteNotifies(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid args") return } - ctx.Err = service.DeleteNotifies(ctx.Username, args.IDs, ctx.Logger) + ctx.Err = service.DeleteNotifies(ctx.UserName, args.IDs, ctx.Logger) } func UpsertSubscription(c *gin.Context) { @@ -78,7 +78,7 @@ func UpsertSubscription(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid subscription args") return } - ctx.Err = service.UpsertSubscription(ctx.Username, args, ctx.Logger) + ctx.Err = service.UpsertSubscription(ctx.UserName, args, ctx.Logger) } func UpdateSubscribe(c *gin.Context) { @@ -96,7 +96,7 @@ func UpdateSubscribe(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid notification type") return } - ctx.Err = service.UpdateSubscribe(ctx.Username, notifytype, args, ctx.Logger) + ctx.Err = service.UpdateSubscribe(ctx.UserName, notifytype, args, ctx.Logger) } func Unsubscribe(c *gin.Context) { @@ -107,12 +107,12 @@ func Unsubscribe(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid notification type") return } - ctx.Err = service.Unsubscribe(ctx.Username, notifytype, ctx.Logger) + ctx.Err = service.Unsubscribe(ctx.UserName, notifytype, ctx.Logger) } func ListSubscriptions(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = service.ListSubscriptions(ctx.Username, ctx.Logger) + ctx.Resp, ctx.Err = service.ListSubscriptions(ctx.UserName, ctx.Logger) } diff --git a/pkg/microservice/aslan/core/system/handler/operation.go b/pkg/microservice/aslan/core/system/handler/operation.go index d4d97b7751ef848d9d7935eb8cdaa22a6b1af2e8..10547c2ddf096a15b82f68c3a3fd8aec52c3e5bf 100644 --- a/pkg/microservice/aslan/core/system/handler/operation.go +++ b/pkg/microservice/aslan/core/system/handler/operation.go @@ -51,7 +51,7 @@ func GetOperationLogs(c *gin.Context) { args := &service.OperationLogArgs{ Username: c.Query("username"), - ProductName: c.Query("product_name"), + ProductName: c.Query("projectName"), Function: c.Query("function"), Status: status, PerPage: perPage, diff --git a/pkg/microservice/aslan/core/system/handler/private_key.go b/pkg/microservice/aslan/core/system/handler/private_key.go index f2b32a1844321895accb26b0df36f6f3b4a4e81d..2043e7a2da91a5b514a0b2c8bb35be228f683e53 100644 --- a/pkg/microservice/aslan/core/system/handler/private_key.go +++ b/pkg/microservice/aslan/core/system/handler/private_key.go @@ -32,11 +32,23 @@ import ( "github.com/koderover/zadig/pkg/tool/log" ) +func ListPrivateKeysInternal(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.ListPrivateKeysInternal(ctx.Logger) +} + func ListPrivateKeys(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = service.ListPrivateKeys(ctx.Logger) + encryptedKey := c.Query("encryptedKey") + if len(encryptedKey) == 0 { + ctx.Err = e.ErrInvalidParam + return + } + ctx.Resp, ctx.Err = service.ListPrivateKeys(encryptedKey, "", c.Query("keyword"), true, ctx.Logger) } func GetPrivateKey(c *gin.Context) { @@ -58,7 +70,7 @@ func CreatePrivateKey(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("CreatePrivateKey json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, "", "新增", "资源管理-主机管理", fmt.Sprintf("hostName:%s ip:%s", args.Name, args.IP), string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, "", "新增", "资源管理-主机管理", fmt.Sprintf("hostName:%s ip:%s", args.Name, args.IP), string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) @@ -66,7 +78,7 @@ func CreatePrivateKey(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid PrivateKey args") return } - args.UpdateBy = ctx.Username + args.UpdateBy = ctx.UserName ctx.Err = service.CreatePrivateKey(args, ctx.Logger) } @@ -83,7 +95,7 @@ func UpdatePrivateKey(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("UpdatePrivateKey json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, "", "更新", "资源管理-主机管理", fmt.Sprintf("hostName:%s ip:%s", args.Name, args.IP), string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, "", "更新", "资源管理-主机管理", fmt.Sprintf("hostName:%s ip:%s", args.Name, args.IP), string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) @@ -91,7 +103,7 @@ func UpdatePrivateKey(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid PrivateKey args") return } - args.UpdateBy = ctx.Username + args.UpdateBy = ctx.UserName ctx.Err = service.UpdatePrivateKey(c.Param("id"), args, ctx.Logger) } @@ -100,6 +112,43 @@ func DeletePrivateKey(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - internalhandler.InsertOperationLog(c, ctx.Username, "", "删除", "资源管理-主机管理", fmt.Sprintf("id:%s", c.Param("id")), "", ctx.Logger) - ctx.Err = service.DeletePrivateKey(c.Param("id"), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, "", "删除", "资源管理-主机管理", fmt.Sprintf("id:%s", c.Param("id")), "", ctx.Logger) + ctx.Err = service.DeletePrivateKey(c.Param("id"), ctx.UserName, ctx.Logger) +} + +func ListLabels(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.ListLabels() +} + +type privateKeyArgs struct { + Option string `json:"option"` + Data []*commonmodels.PrivateKey `json:"data"` +} + +func BatchCreatePrivateKey(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(privateKeyArgs) + data, err := c.GetRawData() + if err != nil { + log.Errorf("batchCreatePrivateKey c.GetRawData() err : %v", err) + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("batchCreatePrivateKey json.Unmarshal err : %v", err) + } + + internalhandler.InsertOperationLog(c, ctx.UserName, "", "批量新增", "资源管理-主机管理", "", string(data), ctx.Logger) + + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + + if err := c.ShouldBindJSON(&args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid PrivateKey args") + return + } + + ctx.Err = service.BatchCreatePrivateKey(args.Data, args.Option, ctx.UserName, ctx.Logger) } diff --git a/pkg/microservice/aslan/core/system/handler/proxy.go b/pkg/microservice/aslan/core/system/handler/proxy.go index 2f0ed2602c4151a951540d1b869642a2f1a538d2..896bca8b10a2284528a99a2138c367d1f380c300 100644 --- a/pkg/microservice/aslan/core/system/handler/proxy.go +++ b/pkg/microservice/aslan/core/system/handler/proxy.go @@ -69,7 +69,7 @@ func CreateProxy(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("CreateProxy json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, "", "新增", "代理", fmt.Sprintf("server:%s:%d", args.Address, args.Port), string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, "", "新增", "代理", fmt.Sprintf("server:%s:%d", args.Address, args.Port), string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) @@ -77,7 +77,7 @@ func CreateProxy(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid proxy args") return } - args.UpdateBy = ctx.Username + args.UpdateBy = ctx.UserName ctx.Err = service.CreateProxy(args, ctx.Logger) } @@ -94,7 +94,7 @@ func UpdateProxy(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("UpdateProxy json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, "", "更新", "代理", fmt.Sprintf("id:%s", args.ID), string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, "", "更新", "代理", fmt.Sprintf("id:%s", args.ID), string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) @@ -102,7 +102,7 @@ func UpdateProxy(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid proxy args") return } - args.UpdateBy = ctx.Username + args.UpdateBy = ctx.UserName ctx.Err = service.UpdateProxy(c.Param("id"), args, ctx.Logger) } @@ -111,7 +111,7 @@ func DeleteProxy(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - internalhandler.InsertOperationLog(c, ctx.Username, "", "删除", "代理", fmt.Sprintf("id:%s", c.Param("id")), "", ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, "", "删除", "代理", fmt.Sprintf("id:%s", c.Param("id")), "", ctx.Logger) ctx.Err = service.DeleteProxy(c.Param("id"), ctx.Logger) } diff --git a/pkg/microservice/aslan/core/system/handler/registry.go b/pkg/microservice/aslan/core/system/handler/registry.go index 61e7f23bf1715c22405f0bd42ef8430a0095152c..23f4f22ea5ea674a2552e7d6f9b46e1a7401defc 100644 --- a/pkg/microservice/aslan/core/system/handler/registry.go +++ b/pkg/microservice/aslan/core/system/handler/registry.go @@ -25,7 +25,6 @@ import ( "github.com/gin-gonic/gin" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" - commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/system/service" internalhandler "github.com/koderover/zadig/pkg/shared/handler" @@ -44,12 +43,14 @@ func GetDefaultRegistryNamespace(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - reg, err := commonservice.GetDefaultRegistryNamespace(ctx.Logger) + reg, _, err := commonservice.FindDefaultRegistry(true, ctx.Logger) if err != nil { ctx.Err = err return } + // FIXME: a new feature in 1.11 added a tls certificate field for registry, but it is not added in this API temporarily + // since it is for Kodespace ONLY ctx.Resp = &Registry{ ID: reg.ID.Hex(), RegAddr: reg.RegAddr, @@ -60,11 +61,44 @@ func GetDefaultRegistryNamespace(c *gin.Context) { } } +func GetRegistryNamespace(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + reg, _, err := commonservice.FindRegistryById(c.Param("id"), false, ctx.Logger) + if err != nil { + ctx.Err = err + return + } + + resp := &Registry{ + ID: reg.ID.Hex(), + RegAddr: reg.RegAddr, + IsDefault: reg.IsDefault, + Namespace: reg.Namespace, + } + + if reg.AdvancedSetting != nil { + resp.AdvancedSetting = &AdvancedRegistrySetting{ + Modified: reg.AdvancedSetting.Modified, + TLSEnabled: reg.AdvancedSetting.TLSEnabled, + TLSCert: reg.AdvancedSetting.TLSCert, + } + } + + ctx.Resp = resp +} + func ListRegistryNamespaces(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = commonservice.ListRegistryNamespaces(ctx.Logger) + encryptedKey := c.Query("encryptedKey") + if len(encryptedKey) == 0 { + ctx.Err = e.ErrInvalidParam + return + } + ctx.Resp, ctx.Err = commonservice.ListRegistryNamespaces(encryptedKey, false, ctx.Logger) } func CreateRegistryNamespace(c *gin.Context) { @@ -79,7 +113,7 @@ func CreateRegistryNamespace(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("CreateRegistryNamespace json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, "", "新增", "系统设置-Registry", fmt.Sprintf("提供商:%s,Namespace:%s", args.RegProvider, args.Namespace), string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, "", "新增", "系统设置-Registry", fmt.Sprintf("提供商:%s,Namespace:%s", args.RegProvider, args.Namespace), string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) if err := c.BindJSON(args); err != nil { @@ -92,7 +126,7 @@ func CreateRegistryNamespace(c *gin.Context) { return } - ctx.Err = service.CreateRegistryNamespace(ctx.Username, args, ctx.Logger) + ctx.Err = service.CreateRegistryNamespace(ctx.UserName, args, ctx.Logger) } func UpdateRegistryNamespace(c *gin.Context) { @@ -107,7 +141,7 @@ func UpdateRegistryNamespace(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("UpdateRegistryNamespace json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, "", "更新", "系统设置-Registry", fmt.Sprintf("提供商:%s,Namespace:%s", args.RegProvider, args.Namespace), string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, "", "更新", "系统设置-Registry", fmt.Sprintf("提供商:%s,Namespace:%s", args.RegProvider, args.Namespace), string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) if err := c.BindJSON(args); err != nil { @@ -120,14 +154,14 @@ func UpdateRegistryNamespace(c *gin.Context) { return } - ctx.Err = service.UpdateRegistryNamespace(ctx.Username, c.Param("id"), args, ctx.Logger) + ctx.Err = service.UpdateRegistryNamespace(ctx.UserName, c.Param("id"), args, ctx.Logger) } func DeleteRegistryNamespace(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - internalhandler.InsertOperationLog(c, ctx.Username, "", "删除", "系统设置-Registry", fmt.Sprintf("registry ID:%s", c.Param("id")), "", ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, "", "删除", "系统设置-Registry", fmt.Sprintf("registry ID:%s", c.Param("id")), "", ctx.Logger) ctx.Err = service.DeleteRegistryNamespace(c.Param("id"), ctx.Logger) } @@ -148,14 +182,14 @@ func ListImages(c *gin.Context) { defer func() { internalhandler.JSONResponse(c, ctx) }() //判断当前registryId是否为空 - regOps := new(commonrepo.FindRegOps) registryID := c.Query("registryId") + var registryInfo *commonmodels.RegistryNamespace + var err error if registryID != "" { - regOps.ID = registryID + registryInfo, _, err = commonservice.FindRegistryById(registryID, false, ctx.Logger) } else { - regOps.IsDefault = true + registryInfo, _, err = commonservice.FindDefaultRegistry(false, ctx.Logger) } - registryInfo, err := service.GetRegistryNamespace(regOps, ctx.Logger) if err != nil { ctx.Logger.Errorf("can't find candidate registry err :%v", err) ctx.Resp = make([]*service.RepoImgResp, 0) @@ -177,9 +211,7 @@ func ListRepoImages(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - regOps := new(commonrepo.FindRegOps) - regOps.IsDefault = true - registryInfo, err := service.GetRegistryNamespace(regOps, ctx.Logger) + registryInfo, _, err := commonservice.FindDefaultRegistry(false, ctx.Logger) if err != nil { ctx.Err = e.ErrInvalidParam.AddErr(err) return diff --git a/pkg/microservice/aslan/core/system/handler/resp.go b/pkg/microservice/aslan/core/system/handler/resp.go index f36bb4da7dc45afd5bc2d6abd14ecf349b06dc33..f62ee99abb09ab4e6af957184882cb9705e7e3b6 100644 --- a/pkg/microservice/aslan/core/system/handler/resp.go +++ b/pkg/microservice/aslan/core/system/handler/resp.go @@ -17,10 +17,17 @@ limitations under the License. package handler type Registry struct { - ID string `json:"id"` - RegAddr string `json:"reg_addr"` - IsDefault bool `json:"is_default"` - Namespace string `json:"namespace"` - AccessKey string `json:"access_key"` - SecretKey string `json:"secret_key"` + ID string `json:"id"` + RegAddr string `json:"reg_addr"` + IsDefault bool `json:"is_default"` + Namespace string `json:"namespace"` + AccessKey string `json:"access_key"` + SecretKey string `json:"secret_key"` + AdvancedSetting *AdvancedRegistrySetting `json:"advanced_setting"` +} + +type AdvancedRegistrySetting struct { + Modified bool `json:"modified"` + TLSEnabled bool `json:"enable_tls"` + TLSCert string `json:"tls_cert"` } diff --git a/pkg/microservice/aslan/core/system/handler/router.go b/pkg/microservice/aslan/core/system/handler/router.go index f85abb7d3d744bfafc4256fd4a327a7607a020e7..e580c1616306fd622028aba9576fd394d88c0a15 100644 --- a/pkg/microservice/aslan/core/system/handler/router.go +++ b/pkg/microservice/aslan/core/system/handler/router.go @@ -18,8 +18,6 @@ package handler import ( "github.com/gin-gonic/gin" - - gin2 "github.com/koderover/zadig/pkg/middleware/gin" ) type Router struct{} @@ -30,18 +28,16 @@ func (*Router) Inject(router *gin.RouterGroup) { proxy.GET("/config", GetProxyConfig) } - router.Use(gin2.Auth()) - // --------------------------------------------------------------------------------------- // 安装脚本管理接口 // --------------------------------------------------------------------------------------- install := router.Group("install") { - install.POST("", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, CreateInstall) - install.PUT("", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, UpdateInstall) + install.POST("", CreateInstall) + install.PUT("", UpdateInstall) install.GET("/:name/:version", GetInstall) install.GET("", ListInstalls) - install.PUT("/delete", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, DeleteInstall) + install.PUT("/delete", DeleteInstall) } // --------------------------------------------------------------------------------------- @@ -51,9 +47,9 @@ func (*Router) Inject(router *gin.RouterGroup) { { proxyManage.GET("", ListProxies) proxyManage.GET("/:id", GetProxy) - proxyManage.POST("", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, CreateProxy) - proxyManage.PUT("/:id", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, UpdateProxy) - proxyManage.DELETE("/:id", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, DeleteProxy) + proxyManage.POST("", CreateProxy) + proxyManage.PUT("/:id", UpdateProxy) + proxyManage.DELETE("/:id", DeleteProxy) proxyManage.POST("/connectionTest", TestConnection) } @@ -63,11 +59,12 @@ func (*Router) Inject(router *gin.RouterGroup) { registry.GET("", ListRegistries) // 获取默认的镜像仓库配置，用于kodespace CLI调用 registry.GET("/namespaces/default", GetDefaultRegistryNamespace) - registry.GET("/namespaces", gin2.RequireSuperAdminAuth, ListRegistryNamespaces) - registry.POST("/namespaces", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, CreateRegistryNamespace) - registry.PUT("/namespaces/:id", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, UpdateRegistryNamespace) + registry.GET("/namespaces/specific/:id", GetRegistryNamespace) + registry.GET("/namespaces", ListRegistryNamespaces) + registry.POST("/namespaces", CreateRegistryNamespace) + registry.PUT("/namespaces/:id", UpdateRegistryNamespace) - registry.DELETE("/namespaces/:id", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, DeleteRegistryNamespace) + registry.DELETE("/namespaces/:id", DeleteRegistryNamespace) registry.GET("/release/repos", ListAllRepos) registry.POST("/images", ListImages) registry.GET("/images/repos/:name", ListRepoImages) @@ -76,10 +73,11 @@ func (*Router) Inject(router *gin.RouterGroup) { s3storage := router.Group("s3storage") { s3storage.GET("", ListS3Storage) - s3storage.POST("", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, CreateS3Storage) + s3storage.POST("", CreateS3Storage) s3storage.GET("/:id", GetS3Storage) - s3storage.PUT("/:id", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, UpdateS3Storage) - s3storage.DELETE("/:id", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, DeleteS3Storage) + s3storage.PUT("/:id", UpdateS3Storage) + s3storage.DELETE("/:id", DeleteS3Storage) + s3storage.POST("/:id/releases/search", ListTars) } //系统清理缓存 @@ -87,6 +85,7 @@ func (*Router) Inject(router *gin.RouterGroup) { { cleanCache.POST("/oneClick", CleanImageCache) cleanCache.GET("/state", CleanCacheState) + cleanCache.POST("/cron", SetCron) } // --------------------------------------------------------------------------------------- @@ -95,8 +94,8 @@ func (*Router) Inject(router *gin.RouterGroup) { github := router.Group("githubApp") { github.GET("", GetGithubApp) - github.POST("", gin2.RequireSuperAdminAuth, CreateGithubApp) - github.DELETE("/:id", gin2.RequireSuperAdminAuth, DeleteGithubApp) + github.POST("", CreateGithubApp) + github.DELETE("/:id", DeleteGithubApp) } // --------------------------------------------------------------------------------------- @@ -104,13 +103,14 @@ func (*Router) Inject(router *gin.RouterGroup) { // --------------------------------------------------------------------------------------- jenkins := router.Group("jenkins") { - jenkins.POST("/integration", gin2.RequireSuperAdminAuth, CreateJenkinsIntegration) + jenkins.GET("/exist", CheckJenkinsIntegration) + jenkins.POST("/integration", CreateJenkinsIntegration) jenkins.GET("/integration", ListJenkinsIntegration) - jenkins.PUT("/integration/:id", gin2.RequireSuperAdminAuth, UpdateJenkinsIntegration) - jenkins.DELETE("/integration/:id", gin2.RequireSuperAdminAuth, DeleteJenkinsIntegration) - jenkins.POST("/user/connection", gin2.RequireSuperAdminAuth, TestJenkinsConnection) - jenkins.GET("/jobNames", gin2.RequireSuperAdminAuth, ListJobNames) - jenkins.GET("/buildArgs/:jobName", gin2.RequireSuperAdminAuth, ListJobBuildArgs) + jenkins.PUT("/integration/:id", UpdateJenkinsIntegration) + jenkins.DELETE("/integration/:id", DeleteJenkinsIntegration) + jenkins.POST("/user/connection", TestJenkinsConnection) + jenkins.GET("/jobNames/:id", ListJobNames) + jenkins.GET("/buildArgs/:id/:jobName", ListJobBuildArgs) } //系统配额 @@ -123,6 +123,20 @@ func (*Router) Inject(router *gin.RouterGroup) { capacity.POST("/clean", CleanCache) } + // workflow concurrency settings + concurrency := router.Group("concurrency") + { + concurrency.GET("/workflow", GetWorkflowConcurrency) + concurrency.POST("/workflow", UpdateWorkflowConcurrency) + } + + // default login default login home page settings + login := router.Group("login") + { + login.GET("/default", GetDefaultLogin) + login.POST("/default", UpdateDefaultLogin) + } + // --------------------------------------------------------------------------------------- // 自定义镜像管理接口 // --------------------------------------------------------------------------------------- @@ -130,9 +144,9 @@ func (*Router) Inject(router *gin.RouterGroup) { { basicImages.GET("", ListBasicImages) basicImages.GET("/:id", GetBasicImage) - basicImages.POST("", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, CreateBasicImage) - basicImages.PUT("/:id", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, UpdateBasicImage) - basicImages.DELETE("/:id", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, DeleteBasicImage) + basicImages.POST("", CreateBasicImage) + basicImages.PUT("/:id", UpdateBasicImage) + basicImages.DELETE("/:id", DeleteBasicImage) } // --------------------------------------------------------------------------------------- @@ -141,9 +155,10 @@ func (*Router) Inject(router *gin.RouterGroup) { integration := router.Group("helm") { integration.GET("", ListHelmRepos) - integration.POST("", gin2.RequireSuperAdminAuth, CreateHelmRepo) - integration.PUT("/:id", gin2.RequireSuperAdminAuth, UpdateHelmRepo) - integration.DELETE("/:id", gin2.RequireSuperAdminAuth, DeleteHelmRepo) + integration.POST("", CreateHelmRepo) + integration.PUT("/:id", UpdateHelmRepo) + integration.DELETE("/:id", DeleteHelmRepo) + integration.GET("/:name/index", ListCharts) } // --------------------------------------------------------------------------------------- @@ -152,10 +167,19 @@ func (*Router) Inject(router *gin.RouterGroup) { privateKey := router.Group("privateKey") { privateKey.GET("", ListPrivateKeys) + privateKey.GET("/internal", ListPrivateKeysInternal) privateKey.GET("/:id", GetPrivateKey) - privateKey.POST("", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, CreatePrivateKey) - privateKey.PUT("/:id", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, UpdatePrivateKey) - privateKey.DELETE("/:id", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, DeletePrivateKey) + privateKey.GET("/labels", ListLabels) + privateKey.POST("", CreatePrivateKey) + privateKey.POST("/batch", BatchCreatePrivateKey) + privateKey.PUT("/:id", UpdatePrivateKey) + privateKey.DELETE("/:id", DeletePrivateKey) + } + + rsaKey := router.Group("rsaKey") + { + rsaKey.GET("publicKey", GetRSAPublicKey) + rsaKey.GET("decryptedText", GetTextFromEncryptedKey) } notification := router.Group("notification") @@ -171,18 +195,53 @@ func (*Router) Inject(router *gin.RouterGroup) { announcement := router.Group("announcement") { - announcement.POST("", gin2.RequireSuperAdminAuth, CreateAnnouncement) - announcement.PUT("/update", gin2.RequireSuperAdminAuth, UpdateAnnouncement) - announcement.GET("/all", gin2.RequireSuperAdminAuth, PullAllAnnouncement) + announcement.POST("", CreateAnnouncement) + announcement.PUT("/update", UpdateAnnouncement) + announcement.GET("/all", PullAllAnnouncement) announcement.GET("", PullNotifyAnnouncement) - announcement.DELETE("/:id", gin2.RequireSuperAdminAuth, DeleteAnnouncement) + announcement.DELETE("/:id", DeleteAnnouncement) } operation := router.Group("operation") { - operation.GET("", gin2.RequireSuperAdminAuth, GetOperationLogs) - operation.POST("", gin2.RequireSuperAdminAuth, AddSystemOperationLog) - operation.PUT("/:id", gin2.RequireSuperAdminAuth, UpdateOperationLog) + operation.GET("", GetOperationLogs) + operation.POST("", AddSystemOperationLog) + operation.PUT("/:id", UpdateOperationLog) } + // --------------------------------------------------------------------------------------- + // system external link + // --------------------------------------------------------------------------------------- + externalLink := router.Group("externalLink") + { + externalLink.GET("", ListExternalLinks) + externalLink.POST("", CreateExternalLink) + externalLink.PUT("/:id", UpdateExternalLink) + externalLink.DELETE("/:id", DeleteExternalLink) + } + + // --------------------------------------------------------------------------------------- + // external system API + // --------------------------------------------------------------------------------------- + externalSystem := router.Group("external") + { + externalSystem.POST("", CreateExternalSystem) + externalSystem.GET("", ListExternalSystem) + externalSystem.GET("/:id", GetExternalSystemDetail) + externalSystem.PUT("/:id", UpdateExternalSystem) + externalSystem.DELETE("/:id", DeleteExternalSystem) + } + + // --------------------------------------------------------------------------------------- + // sonar integration API + // --------------------------------------------------------------------------------------- + sonar := router.Group("sonar") + { + sonar.POST("/integration", CreateSonarIntegration) + sonar.PUT("/integration/:id", UpdateSonarIntegration) + sonar.GET("/integration", ListSonarIntegration) + sonar.GET("/integration/:id", GetSonarIntegration) + sonar.DELETE("/integration/:id", DeleteSonarIntegration) + sonar.POST("/validate", ValidateSonarInformation) + } } diff --git a/pkg/microservice/aslan/core/system/handler/rsa_key.go b/pkg/microservice/aslan/core/system/handler/rsa_key.go new file mode 100644 index 0000000000000000000000000000000000000000..74b9f01d046c499e44caede8fe99abbb3b53d02f --- /dev/null +++ b/pkg/microservice/aslan/core/system/handler/rsa_key.go @@ -0,0 +1,42 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +func GetRSAPublicKey(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + ctx.Resp, ctx.Err = service.GetRSAPublicKey() +} + +func GetTextFromEncryptedKey(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + encryptedKey := c.Query("encryptedKey") + if len(encryptedKey) == 0 { + ctx.Err = e.ErrInvalidParam + return + } + ctx.Resp, ctx.Err = service.GetAesKeyFromEncryptedKey(encryptedKey, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/system/handler/s3.go b/pkg/microservice/aslan/core/system/handler/s3.go index 8c2c573236b68221594f983f009d779a437e8ec1..53ff2db198a0e9d297da70f063c9d8251303d1b8 100644 --- a/pkg/microservice/aslan/core/system/handler/s3.go +++ b/pkg/microservice/aslan/core/system/handler/s3.go @@ -36,7 +36,12 @@ func ListS3Storage(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = service.ListS3Storage(ctx.Logger) + encryptedKey := c.Query("encryptedKey") + if len(encryptedKey) == 0 { + ctx.Err = e.ErrInvalidParam + return + } + ctx.Resp, ctx.Err = service.ListS3Storage(encryptedKey, ctx.Logger) } func CreateS3Storage(c *gin.Context) { @@ -51,7 +56,7 @@ func CreateS3Storage(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("CreateS3Storage json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, "", "新增", "系统设置-对象存储", fmt.Sprintf("地址:%s", c.GetString("s3StorageEndpoint")), string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, "", "新增", "系统设置-对象存储", fmt.Sprintf("地址:%s", c.GetString("s3StorageEndpoint")), string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) if err := c.BindJSON(args); err != nil { @@ -67,7 +72,7 @@ func CreateS3Storage(c *gin.Context) { return } - ctx.Err = service.CreateS3Storage(ctx.Username, args, ctx.Logger) + ctx.Err = service.CreateS3Storage(ctx.UserName, args, ctx.Logger) } func GetS3Storage(c *gin.Context) { @@ -89,7 +94,7 @@ func UpdateS3Storage(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("UpdateS3Storage json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, "", "更新", "系统设置-对象存储", fmt.Sprintf("地址:%s", c.GetString("s3StorageEndpoint")), string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, "", "更新", "系统设置-对象存储", fmt.Sprintf("地址:%s", c.GetString("s3StorageEndpoint")), string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) if err := c.BindJSON(args); err != nil { @@ -106,14 +111,31 @@ func UpdateS3Storage(c *gin.Context) { } id := c.Param("id") - ctx.Err = service.UpdateS3Storage(ctx.Username, id, args, ctx.Logger) + ctx.Err = service.UpdateS3Storage(ctx.UserName, id, args, ctx.Logger) } func DeleteS3Storage(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - internalhandler.InsertOperationLog(c, ctx.Username, "", "删除", "系统设置-对象存储", fmt.Sprintf("s3Storage ID:%s", c.Param("id")), "", ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, "", "删除", "系统设置-对象存储", fmt.Sprintf("s3Storage ID:%s", c.Param("id")), "", ctx.Logger) + + ctx.Err = service.DeleteS3Storage(ctx.UserName, c.Param("id"), ctx.Logger) +} + +type ListTarsOption struct { + Names []string `json:"names"` +} + +func ListTars(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(ListTarsOption) + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } - ctx.Err = service.DeleteS3Storage(ctx.Username, c.Param("id"), ctx.Logger) + ctx.Resp, ctx.Err = service.ListTars(c.Param("id"), c.Query("kind"), args.Names, ctx.Logger) } diff --git a/pkg/microservice/aslan/core/system/handler/sonar.go b/pkg/microservice/aslan/core/system/handler/sonar.go new file mode 100644 index 0000000000000000000000000000000000000000..f98896612468149188d52da303eecc2d2892bf3e --- /dev/null +++ b/pkg/microservice/aslan/core/system/handler/sonar.go @@ -0,0 +1,179 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "encoding/json" + "errors" + "fmt" + "github.com/gin-gonic/gin" + "github.com/koderover/zadig/pkg/tool/crypto" + + commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/system/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/log" +) + +var SonarIntegrationValidationError = errors.New("name and server must be provided") + +func CreateSonarIntegration(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(service.SonarIntegration) + + data, err := c.GetRawData() + if err != nil { + log.Errorf("Create sonar integration GetRawData err : %s", err) + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("Create sonar integration Unmarshal err : %s", err) + } + internalhandler.InsertOperationLog(c, ctx.UserName, "", "新增", "系统配置-Sonar集成", fmt.Sprintf("server: %s, token: %s", args.ServerAddress, args.Token), string(data), ctx.Logger) + + if err != nil { + ctx.Err = fmt.Errorf("failed to update sonar integration: %s", err) + return + } + + if args.ServerAddress == "" || args.Token == "" { + ctx.Err = SonarIntegrationValidationError + return + } + ctx.Err = service.CreateSonarIntegration(args, ctx.Logger) +} + +func UpdateSonarIntegration(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(service.SonarIntegration) + + data, err := c.GetRawData() + if err != nil { + log.Errorf("Update sonar integration GetRawData err : %s", err) + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("Update sonar integration Unmarshal err : %s", err) + } + internalhandler.InsertOperationLog(c, ctx.UserName, "", "更新", "系统配置-Sonar集成", fmt.Sprintf("server: %s, token: %s", args.ServerAddress, args.Token), string(data), ctx.Logger) + + if err != nil { + ctx.Err = fmt.Errorf("failed to update sonar integration: %s", err) + return + } + + if args.ServerAddress == "" || args.Token == "" { + ctx.Err = SonarIntegrationValidationError + return + } + ctx.Err = service.UpdateSonarIntegration(c.Param("id"), args, ctx.Logger) +} + +func ListSonarIntegration(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + encryptedKey := c.Query("encryptedKey") + if len(encryptedKey) == 0 { + ctx.Err = e.ErrInvalidParam + return + } + + aesKey, err := commonservice.GetAesKeyFromEncryptedKey(encryptedKey, ctx.Logger) + if err != nil { + ctx.Err = err + return + } + + sonarList, _, err := service.ListSonarIntegration(ctx.Logger) + if err != nil { + ctx.Err = err + return + } + + for _, sonar := range sonarList { + encryptedSonarToken, err := crypto.AesEncryptByKey(sonar.Token, aesKey.PlainText) + if err != nil { + ctx.Err = fmt.Errorf("failed to encrypt sonar token, err: %s", err) + return + } + sonar.Token = encryptedSonarToken + } + ctx.Resp = sonarList +} + +func GetSonarIntegration(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + encryptedKey := c.Query("encryptedKey") + if len(encryptedKey) == 0 { + ctx.Err = e.ErrInvalidParam + return + } + + aesKey, err := commonservice.GetAesKeyFromEncryptedKey(encryptedKey, ctx.Logger) + if err != nil { + ctx.Err = err + return + } + + resp, err := service.GetSonarIntegration(c.Param("id"), ctx.Logger) + if err != nil { + ctx.Err = err + return + } + encryptedSonarToken, err := crypto.AesEncryptByKey(resp.Token, aesKey.PlainText) + if err != nil { + ctx.Err = fmt.Errorf("failed to encrypt sonar token, err: %s", err) + return + } + resp.Token = encryptedSonarToken + ctx.Resp = resp +} + +func DeleteSonarIntegration(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + internalhandler.InsertOperationLog(c, ctx.UserName, "", "删除", "系统配置-Sonar集成", fmt.Sprintf("id:%s", c.Param("id")), "", ctx.Logger) + ctx.Err = service.DeleteSonarIntegration(c.Param("id"), ctx.Logger) +} + +func ValidateSonarInformation(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(service.SonarIntegration) + + data, err := c.GetRawData() + if err != nil { + log.Errorf("Validate sonar integration GetRawData err : %s", err) + ctx.Err = fmt.Errorf("validate sonar integration GetRawData err : %s", err) + return + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("Validate sonar integration Unmarshal err : %s", err) + ctx.Err = fmt.Errorf("validate sonar integration Unmarshal err : %s", err) + return + } + + ctx.Err = service.ValidateSonarIntegration(args, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/system/handler/system_cache.go b/pkg/microservice/aslan/core/system/handler/system_cache.go index 00e87f17221d441ee5143cf43f9f4ff31c329939..ced5f6e75e2081f3ae99d002ef144c64a4367e1a 100644 --- a/pkg/microservice/aslan/core/system/handler/system_cache.go +++ b/pkg/microservice/aslan/core/system/handler/system_cache.go @@ -30,6 +30,23 @@ func CleanImageCache(c *gin.Context) { ctx.Err = service.CleanImageCache(ctx.Logger) } +func SetCron(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + arg := new(SetCleanConfig) + if err := c.ShouldBindJSON(arg); err != nil { + ctx.Err = err + return + } + ctx.Err = service.SetCron(arg.Cron, arg.CronEnabled, ctx.Logger) +} + +type SetCleanConfig struct { + Cron string `json:"cron"` + CronEnabled bool `json:"cron_enabled"` +} + func CleanCacheState(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() diff --git a/pkg/microservice/aslan/core/system/service/basic_images.go b/pkg/microservice/aslan/core/system/service/basic_images.go index 1fe3c19d7c169acfce3c622d5ec795006d317890..769b17b23e2fa552755d663b60cd20bfae620eae 100644 --- a/pkg/microservice/aslan/core/system/service/basic_images.go +++ b/pkg/microservice/aslan/core/system/service/basic_images.go @@ -55,6 +55,14 @@ func InitbasicImageInfos() []*commonmodels.BasicImage { UpdateTime: time.Now().Unix(), UpdateBy: "system", }, + { + Value: "sonarsource/sonar-scanner-cli", + Label: "sonar:latest", + CreateTime: time.Now().Unix(), + UpdateTime: time.Now().Unix(), + UpdateBy: "system", + ImageType: "sonar", + }, } return basicImageInfos @@ -69,9 +77,10 @@ func GetBasicImage(id string, log *zap.SugaredLogger) (*commonmodels.BasicImage, return resp, nil } -func ListBasicImages(imageFrom string, log *zap.SugaredLogger) ([]*commonmodels.BasicImage, error) { +func ListBasicImages(imageFrom string, imageType string, log *zap.SugaredLogger) ([]*commonmodels.BasicImage, error) { opt := &commonrepo.BasicImageOpt{ ImageFrom: imageFrom, + ImageType: imageType, } resp, err := commonrepo.NewBasicImageColl().List(opt) if err != nil { diff --git a/pkg/microservice/aslan/core/system/service/capacity.go b/pkg/microservice/aslan/core/system/service/capacity.go index fba668bfc07a1ac88ecf1279c5a137f6d7dffd98..4da6d2c377fd2e07f1babac7b9138ecfd4648a04 100644 --- a/pkg/microservice/aslan/core/system/service/capacity.go +++ b/pkg/microservice/aslan/core/system/service/capacity.go @@ -25,27 +25,13 @@ import ( commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/notify" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/poetry" "github.com/koderover/zadig/pkg/tool/log" s3tool "github.com/koderover/zadig/pkg/tool/s3" "github.com/koderover/zadig/pkg/util" ) -const ( - defaultWorkflowMaxDays int = 365 - //logTag = "SysCap" -) - -var defaultWorkflowTaskRetention = &commonmodels.CapacityStrategy{ - Target: commonmodels.WorkflowTaskRetention, - Retention: &commonmodels.RetentionConfig{ - MaxDays: defaultWorkflowMaxDays, - }, -} - func UpdateSysCapStrategy(strategy *commonmodels.CapacityStrategy) error { if err := validateStrategy(strategy); err != nil { return err @@ -65,7 +51,7 @@ func UpdateSysCapStrategy(strategy *commonmodels.CapacityStrategy) error { func GetCapacityStrategy(target commonmodels.CapacityTarget) (*commonmodels.CapacityStrategy, error) { result, err := commonrepo.NewStrategyColl().GetByTarget(target) if err != nil && target == commonmodels.WorkflowTaskRetention { - return defaultWorkflowTaskRetention, nil // Return default setup + return commonmodels.DefaultWorkflowTaskRetention, nil // Return default setup } return result, err } @@ -74,7 +60,7 @@ func HandleSystemGC(dryRun bool) error { // Find the strategy strategy, err := commonrepo.NewStrategyColl().GetByTarget(commonmodels.WorkflowTaskRetention) if err != nil { - strategy = defaultWorkflowTaskRetention + strategy = commonmodels.DefaultWorkflowTaskRetention } else if err = validateStrategy(strategy); err != nil { return err } @@ -172,20 +158,6 @@ func sendSyscapNotify(handleErr error, totalCleanTasks *int) { if handleErr != nil { content.Content = fmt.Sprintf("清理时间: %s, 状态: 失败, 内容: %v", now, handleErr) } - - notifyInfo := &commonmodels.Notify{ - Type: config.Message, - Content: content, - CreateTime: time.Now().Unix(), - IsRead: false, - } - - poetryClient := poetry.New(config.PoetryAPIServer(), config.PoetryAPIRootKey()) - users, _ := poetryClient.ListProductPermissionUsers("", "", log.SugaredLogger()) - for _, user := range users { - notifyInfo.Receiver = user - notify.NewNotifyClient().CreateNotify(user, notifyInfo) - } } func handleWorkflowTaskRetentionCenter(strategy *commonmodels.CapacityStrategy, dryRun bool) error { @@ -196,17 +168,25 @@ func handleWorkflowTaskRetentionCenter(strategy *commonmodels.CapacityStrategy, defer sendSyscapNotify(handleErr, &totalCleanTasks) // Figuring out starting option based on capacity strategy - var option *commonrepo.ListAllTaskOption const batch = 100 retention := strategy.Retention if retention.MaxDays > 0 { retentionTime := time.Now().AddDate(0, 0, -retention.MaxDays).Unix() - option = &commonrepo.ListAllTaskOption{ + option := &commonrepo.ListAllTaskOption{ BeforeCreatTime: true, CreateTime: retentionTime, Limit: batch, } totalCleanTasks, handleErr = handleWorkflowTaskRetention(dryRun, batch, option) + if handleErr != nil { + return handleErr + } + v4Option := &commonrepo.ListWorkflowTaskV4Option{ + BeforeCreatTime: true, + CreateTime: retentionTime, + Limit: batch, + } + totalCleanTasks, handleErr = handleWorkflowTaskV4Retention(dryRun, batch, v4Option) return handleErr } @@ -219,7 +199,7 @@ func handleWorkflowTaskRetentionCenter(strategy *commonmodels.CapacityStrategy, return err } for _, workflow := range workflows { - option = &commonrepo.ListAllTaskOption{ + option := &commonrepo.ListAllTaskOption{ ProductName: workflow.ProductTmplName, PipelineName: workflow.Name, Type: config.WorkflowType, @@ -233,6 +213,26 @@ func handleWorkflowTaskRetentionCenter(strategy *commonmodels.CapacityStrategy, totalCleanTasks += count } + // clean workflow task v4 data + workflowV4s, _, err := commonrepo.NewWorkflowV4Coll().List(&commonrepo.ListWorkflowV4Option{}, 0, 0) + if err != nil { + log.Errorf("list workflowV4s failed, err:%v", err) + handleErr = err + return err + } + for _, workflow := range workflowV4s { + option := &commonrepo.ListWorkflowTaskV4Option{ + WorkflowName: workflow.Name, + Skip: retention.MaxItems, + Limit: batch, + } + count, err := handleWorkflowTaskV4Retention(dryRun, batch, option) + if err != nil { + continue + } + totalCleanTasks += count + } + // 清理服务工作流任务数据 pipelines, err := commonrepo.NewPipelineColl().List(&commonrepo.PipelineListOption{}) if err != nil { @@ -241,7 +241,7 @@ func handleWorkflowTaskRetentionCenter(strategy *commonmodels.CapacityStrategy, return err } for _, pipeline := range pipelines { - option = &commonrepo.ListAllTaskOption{ + option := &commonrepo.ListAllTaskOption{ ProductName: pipeline.ProductName, PipelineName: pipeline.Name, Type: config.SingleType, @@ -263,7 +263,7 @@ func handleWorkflowTaskRetentionCenter(strategy *commonmodels.CapacityStrategy, return err } for _, testing := range testings { - option = &commonrepo.ListAllTaskOption{ + option := &commonrepo.ListAllTaskOption{ ProductName: testing.ProductName, PipelineName: fmt.Sprintf("%s-%s", testing.Name, "job"), Type: config.TestType, @@ -314,6 +314,33 @@ func handleWorkflowTaskRetention(dryRun bool, batch int, option *commonrepo.List return len(removeIds), nil } +func handleWorkflowTaskV4Retention(dryRun bool, batch int, option *commonrepo.ListWorkflowTaskV4Option) (int, error) { + s3Server, _ := s3.FindDefaultS3() + // Clean up in batches to prevent pressure of memory spike + var removeIds []string + + for { + staleTasks, _, err := commonrepo.NewworkflowTaskv4Coll().List(option) + if err != nil { + return 0, err + } + if len(staleTasks) == 0 { + break + } + ids := cleanStaleTaskV4s(staleTasks, s3Server, dryRun) + if ids != nil { + removeIds = append(removeIds, ids...) + } + if len(staleTasks) < batch { // last batch + break + } + option.Skip += batch + } + + log.Infof("%d stale workflow tasks will be cleaned up", len(removeIds)) + return len(removeIds), nil +} + // //func (s *Service) logInfo(format string, args ...interface{}) { // s.logger.Infof("[%v]: %v", logTag, fmt.Sprintf(format, args...)) @@ -340,6 +367,24 @@ func cleanStaleTasks(tasks []*task.Task, s3Server *s3.S3, dryRun bool) []string return ids } +func cleanStaleTaskV4s(tasks []*commonmodels.WorkflowTask, s3Server *s3.S3, dryRun bool) []string { + ids := make([]string, len(tasks)) + paths := make([]string, len(tasks)) + for i, task := range tasks { + ids[i] = task.ID.Hex() + paths[i] = fmt.Sprintf("%s/%d/", task.WorkflowName, task.TaskID) + } + forcedPathStyle := true + if s3Server.Provider == setting.ProviderSourceAli { + forcedPathStyle = false + } + s3client, err := s3tool.NewClient(s3Server.Endpoint, s3Server.Ak, s3Server.Sk, s3Server.Insecure, forcedPathStyle) + if err == nil { + go s3client.RemoveFiles(s3Server.Bucket, paths) + } + return ids +} + func validateStrategy(strategy *commonmodels.CapacityStrategy) error { if strategy.Target == commonmodels.WorkflowTaskRetention { retention := strategy.Retention diff --git a/pkg/microservice/aslan/core/system/service/concurrency.go b/pkg/microservice/aslan/core/system/service/concurrency.go new file mode 100644 index 0000000000000000000000000000000000000000..38a56d451c6941ea2decf675c1a0f89498f85638 --- /dev/null +++ b/pkg/microservice/aslan/core/system/service/concurrency.go @@ -0,0 +1,63 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "errors" + + "go.uber.org/zap" + + configbase "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + workflowservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" + "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/kube/updater" +) + +func GetWorkflowConcurrency() (*WorkflowConcurrencySettings, error) { + configuration, err := commonrepo.NewSystemSettingColl().Get() + if err != nil { + return nil, err + } + return &WorkflowConcurrencySettings{ + WorkflowConcurrency: configuration.WorkflowConcurrency, + BuildConcurrency: configuration.BuildConcurrency, + }, nil +} + +func UpdateWorkflowConcurrency(workflowConcurrency, buildConcurrency int64, log *zap.SugaredLogger) error { + // check if there are running tasks + tasks := workflowservice.RunningPipelineTasks() + if len(tasks) > 0 { + return errors.New("workflow settings must be set when NO task is running") + } + // first update system configuration table + err := commonrepo.NewSystemSettingColl().UpdateConcurrencySetting(workflowConcurrency, buildConcurrency) + if err != nil { + log.Errorf("Failed to update system settings, the error is: %s", err) + return err + } + // then we update the warpdrive deployment, where the workflow concurrency happens + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), setting.LocalClusterID) + if err != nil { + return e.ErrScaleService.AddErr(err) + } + return updater.ScaleDeployment(config.Namespace(), configbase.WarpDriveServiceName(), int(workflowConcurrency), kubeClient) +} diff --git a/pkg/microservice/aslan/core/system/service/external_link.go b/pkg/microservice/aslan/core/system/service/external_link.go new file mode 100644 index 0000000000000000000000000000000000000000..812e3a01edf1e360fca5094d2ae4cc939d056723 --- /dev/null +++ b/pkg/microservice/aslan/core/system/service/external_link.go @@ -0,0 +1,61 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "go.uber.org/zap" + + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +func ListExternalLinks(log *zap.SugaredLogger) ([]*commonmodels.ExternalLink, error) { + resp, err := commonrepo.NewExternalLinkColl().List() + if err != nil { + log.Errorf("ExternalLink.List error: %s", err) + return resp, e.ErrListExternalLink.AddErr(err) + } + return resp, nil +} + +func CreateExternalLink(args *commonmodels.ExternalLink, log *zap.SugaredLogger) error { + err := commonrepo.NewExternalLinkColl().Create(args) + if err != nil { + log.Errorf("ExternalLink.Create error: %s", err) + return e.ErrCreateExternalLink.AddErr(err) + } + return nil +} + +func UpdateExternalLink(id string, args *commonmodels.ExternalLink, log *zap.SugaredLogger) error { + err := commonrepo.NewExternalLinkColl().Update(id, args) + if err != nil { + log.Errorf("ExternalLink.Update %s error: %s", id, err) + return e.ErrUpdateExternalLink.AddErr(err) + } + return nil +} + +func DeleteExternalLink(id string, log *zap.SugaredLogger) error { + err := commonrepo.NewExternalLinkColl().Delete(id) + if err != nil { + log.Errorf("ExternalLink.Delete %s error: %s", id, err) + return e.ErrDeleteExternalLink.AddErr(err) + } + return nil +} diff --git a/pkg/microservice/aslan/core/system/service/external_system.go b/pkg/microservice/aslan/core/system/service/external_system.go new file mode 100644 index 0000000000000000000000000000000000000000..c95bff1447e3551d08f8b87f7cf6ad81192335bd --- /dev/null +++ b/pkg/microservice/aslan/core/system/service/external_system.go @@ -0,0 +1,103 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "go.uber.org/zap" + + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/tool/crypto" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +func CreateExternalSystem(args *ExternalSystemDetail, log *zap.SugaredLogger) error { + err := commonrepo.NewExternalSystemColl().Create(&commonmodels.ExternalSystem{ + Name: args.Name, + Server: args.Server, + APIToken: args.APIToken, + }) + if err != nil { + log.Errorf("Create external system error: %s", err) + return e.ErrCreateExternalLink.AddErr(err) + } + return nil +} + +func ListExternalSystem(encryptedKey string, pageNum, pageSize int64, log *zap.SugaredLogger) ([]*ExternalSystemDetail, int64, error) { + resp, length, err := commonrepo.NewExternalSystemColl().List(pageNum, pageSize) + if err != nil { + log.Errorf("Failed to list external system from db, the error is: %s", err) + return nil, 0, err + } + aesKey, err := service.GetAesKeyFromEncryptedKey(encryptedKey, log) + if err != nil { + return nil, 0, err + } + systemList := make([]*ExternalSystemDetail, 0) + for _, item := range resp { + apiToken, err := crypto.AesEncryptByKey(item.APIToken, aesKey.PlainText) + if err != nil { + return nil, 0, err + } + systemList = append(systemList, &ExternalSystemDetail{ + ID: item.ID.Hex(), + Name: item.Name, + Server: item.Server, + APIToken: apiToken, + }) + } + return systemList, length, nil +} + +func GetExternalSystemDetail(id string, log *zap.SugaredLogger) (*ExternalSystemDetail, error) { + resp := new(ExternalSystemDetail) + externalSystem, err := commonrepo.NewExternalSystemColl().GetByID(id) + if err != nil { + log.Errorf("Failed to get external system detail from id %s, the error is: %s", id, err) + return nil, err + } + resp.ID = externalSystem.ID.Hex() + resp.Name = externalSystem.Name + resp.Server = externalSystem.Server + resp.APIToken = externalSystem.APIToken + return resp, nil +} + +func UpdateExternalSystem(id string, system *ExternalSystemDetail, log *zap.SugaredLogger) error { + err := commonrepo.NewExternalSystemColl().Update( + id, + &commonmodels.ExternalSystem{ + Name: system.Name, + Server: system.Server, + APIToken: system.APIToken, + }, + ) + if err != nil { + log.Errorf("update external system error: %s", err) + } + return err +} + +func DeleteExternalSystem(id string, log *zap.SugaredLogger) error { + err := commonrepo.NewExternalSystemColl().DeleteByID(id) + if err != nil { + log.Errorf("Failed to delete external system of id: %s, the error is: %s", id, err) + } + return err +} diff --git a/pkg/microservice/aslan/core/system/service/helm.go b/pkg/microservice/aslan/core/system/service/helm.go index 585e2f5f3da2923c82acbb51afe9c7ae9fc1e58a..cdcf12bc50f216bcf71a1a0607ae8bdc4508dbde 100644 --- a/pkg/microservice/aslan/core/system/service/helm.go +++ b/pkg/microservice/aslan/core/system/service/helm.go @@ -21,8 +21,19 @@ import ( commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/tool/helmclient" ) +type IndexFileResp struct { + Entries map[string][]*ChartVersion `json:"entries"` +} + +type ChartVersion struct { + ChartName string `json:"chartName"` + Version string `json:"version"` +} + func ListHelmRepos(log *zap.SugaredLogger) ([]*commonmodels.HelmRepo, error) { helmRepos, err := commonrepo.NewHelmRepoColl().List() if err != nil { @@ -56,3 +67,34 @@ func DeleteHelmRepo(id string, log *zap.SugaredLogger) error { } return nil } + +func ListCharts(name string, log *zap.SugaredLogger) (*IndexFileResp, error) { + chartRepo, err := commonrepo.NewHelmRepoColl().Find(&commonrepo.HelmRepoFindOption{RepoName: name}) + if err != nil { + return nil, err + } + + client, err := helmclient.NewClient() + if err != nil { + return nil, err + } + + indexInfo, err := client.FetchIndexYaml(service.GeneHelmRepo(chartRepo)) + if err != nil { + return nil, err + } + + indexResp := &IndexFileResp{ + Entries: make(map[string][]*ChartVersion), + } + + for name, entries := range indexInfo.Entries { + for _, chart := range entries { + indexResp.Entries[name] = append(indexResp.Entries[name], &ChartVersion{ + ChartName: chart.Name, + Version: chart.Version, + }) + } + } + return indexResp, nil +} diff --git a/pkg/microservice/aslan/core/system/service/jenkins.go b/pkg/microservice/aslan/core/system/service/jenkins.go index 65bae1a824dc9a5a749cc99be9b631f0a32f1517..8e7815c5d50d860091bffbbf385da2657fc4d2ec 100644 --- a/pkg/microservice/aslan/core/system/service/jenkins.go +++ b/pkg/microservice/aslan/core/system/service/jenkins.go @@ -25,7 +25,10 @@ import ( commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/tool/crypto" e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/types" ) type JenkinsArgs struct { @@ -37,6 +40,7 @@ type JenkinsArgs struct { type JenkinsBuildArgs struct { Name string `json:"name"` Value interface{} `json:"value"` + Type string `json:"type"` } func CreateJenkinsIntegration(args *commonmodels.JenkinsIntegration, log *zap.SugaredLogger) error { @@ -47,13 +51,29 @@ func CreateJenkinsIntegration(args *commonmodels.JenkinsIntegration, log *zap.Su return nil } -func ListJenkinsIntegration(log *zap.SugaredLogger) ([]*commonmodels.JenkinsIntegration, error) { +func ListJenkinsIntegration(encryptedKey string, log *zap.SugaredLogger) ([]*commonmodels.JenkinsIntegration, error) { jenkinsIntegrations, err := commonrepo.NewJenkinsIntegrationColl().List() if err != nil { log.Errorf("ListJenkinsIntegration err:%v", err) return []*commonmodels.JenkinsIntegration{}, e.ErrListJenkinsIntegration.AddErr(err) } + if len(encryptedKey) == 0 { + return jenkinsIntegrations, nil + } + + aesKey, err := service.GetAesKeyFromEncryptedKey(encryptedKey, log) + if err != nil { + log.Errorf("ListJenkinsIntegration GetAesKeyFromEncryptedKey err:%v", err) + return nil, err + } + for _, integration := range jenkinsIntegrations { + integration.Password, err = crypto.AesEncryptByKey(integration.Password, aesKey.PlainText) + if err != nil { + log.Errorf("ListJenkinsIntegration AesEncryptByKey err:%v", err) + return nil, err + } + } return jenkinsIntegrations, nil } @@ -83,24 +103,22 @@ func TestJenkinsConnection(args *JenkinsArgs, log *zap.SugaredLogger) error { return nil } -func getJenkinsClient(log *zap.SugaredLogger) (*gojenkins.Jenkins, context.Context, error) { +func getJenkinsClient(id string, log *zap.SugaredLogger) (*gojenkins.Jenkins, context.Context, error) { ctx := context.Background() - jenkinsIntegrations, err := ListJenkinsIntegration(log) + jenkinsIntegration, err := commonrepo.NewJenkinsIntegrationColl().Get(id) if err != nil { - return nil, ctx, err - } - if len(jenkinsIntegrations) == 0 { return nil, ctx, fmt.Errorf("未找到jenkins集成数据") } - jenkinsClient, err := gojenkins.CreateJenkins(nil, jenkinsIntegrations[0].URL, jenkinsIntegrations[0].Username, jenkinsIntegrations[0].Password).Init(ctx) + + jenkinsClient, err := gojenkins.CreateJenkins(nil, jenkinsIntegration.URL, jenkinsIntegration.Username, jenkinsIntegration.Password).Init(ctx) if err != nil { return nil, ctx, err } return jenkinsClient, ctx, nil } -func ListJobNames(log *zap.SugaredLogger) ([]string, error) { - jenkinsClient, ctx, err := getJenkinsClient(log) +func ListJobNames(id string, log *zap.SugaredLogger) ([]string, error) { + jenkinsClient, ctx, err := getJenkinsClient(id, log) if err != nil { return []string{}, e.ErrListJobNames.AddErr(err) } @@ -115,8 +133,8 @@ func ListJobNames(log *zap.SugaredLogger) ([]string, error) { return jobNames, nil } -func ListJobBuildArgs(jobName string, log *zap.SugaredLogger) ([]*JenkinsBuildArgs, error) { - jenkinsClient, ctx, err := getJenkinsClient(log) +func ListJobBuildArgs(id, jobName string, log *zap.SugaredLogger) ([]*JenkinsBuildArgs, error) { + jenkinsClient, ctx, err := getJenkinsClient(id, log) if err != nil { return []*JenkinsBuildArgs{}, e.ErrListJobBuildArgs.AddErr(err) } @@ -130,10 +148,17 @@ func ListJobBuildArgs(jobName string, log *zap.SugaredLogger) ([]*JenkinsBuildAr } jenkinsBuildArgsResp := make([]*JenkinsBuildArgs, 0) for _, paramDefinition := range paramDefinitions { - jenkinsBuildArgsResp = append(jenkinsBuildArgsResp, &JenkinsBuildArgs{ + arg := &JenkinsBuildArgs{ Name: paramDefinition.DefaultParameterValue.Name, Value: paramDefinition.DefaultParameterValue.Value, - }) + Type: paramDefinition.Type, + } + if paramDefinition.Type == "ChoiceParameterDefinition" { + arg.Type = string(types.Choice) + } else { + arg.Type = string(types.Str) + } + jenkinsBuildArgsResp = append(jenkinsBuildArgsResp, arg) } return jenkinsBuildArgsResp, nil } diff --git a/pkg/microservice/aslan/core/system/service/login.go b/pkg/microservice/aslan/core/system/service/login.go new file mode 100644 index 0000000000000000000000000000000000000000..fd8de30e7e55a1fd415a9320db2e940ad9c2fee9 --- /dev/null +++ b/pkg/microservice/aslan/core/system/service/login.go @@ -0,0 +1,51 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "go.uber.org/zap" + + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/setting" +) + +func GetDefaultLogin(logger *zap.SugaredLogger) (*GetDefaultLoginResponse, error) { + configuration, err := commonrepo.NewSystemSettingColl().Get() + if err != nil { + logger.Errorf("GetDefaultLogin error:%s", err) + return nil, err + } + defaultLogin := setting.DefaultLoginLocal + if len(configuration.DefaultLogin) > 0 { + defaultLogin = configuration.DefaultLogin + } + return &GetDefaultLoginResponse{ + DefaultLogin: defaultLogin, + }, nil +} + +type GetDefaultLoginResponse struct { + DefaultLogin string `json:"default_login"` +} + +type UpdateDefaultLoginParams struct { + DefaultLogin string `json:"default_login"` +} + +func UpdateDefaultLogin(defaultLogin string, _ *zap.SugaredLogger) error { + return commonrepo.NewSystemSettingColl().UpdateDefaultLoginSetting(defaultLogin) +} diff --git a/pkg/microservice/aslan/core/system/service/private_key.go b/pkg/microservice/aslan/core/system/service/private_key.go index d34ca4a00ea76201f8eeb0bf027203e5d48664ee..b9bc0b84cc86b6c152c1f7ad2203ef788848c9f2 100644 --- a/pkg/microservice/aslan/core/system/service/private_key.go +++ b/pkg/microservice/aslan/core/system/service/private_key.go @@ -17,17 +17,64 @@ limitations under the License. package service import ( + "fmt" + "strings" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/util/sets" "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/pm" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/crypto" e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/types" ) -func ListPrivateKeys(log *zap.SugaredLogger) ([]*commonmodels.PrivateKey, error) { +func ListPrivateKeys(encryptedKey, projectName, keyword string, systemOnly bool, log *zap.SugaredLogger) ([]*commonmodels.PrivateKey, error) { + var resp []*commonmodels.PrivateKey + var err error + privateKeys, err := commonrepo.NewPrivateKeyColl().List(&commonrepo.PrivateKeyArgs{ProjectName: projectName, SystemOnly: systemOnly}) + if err != nil { + log.Errorf("PrivateKey.List error: %s", err) + return resp, e.ErrListPrivateKeys + } + + if keyword == "" { + resp = privateKeys + } else { + for _, privateKey := range privateKeys { + if strings.Contains(privateKey.Name, keyword) || strings.Contains(privateKey.IP, keyword) { + resp = append(resp, privateKey) + } + } + } + + aesKey, err := commonservice.GetAesKeyFromEncryptedKey(encryptedKey, log) + if err != nil { + return nil, err + } + for _, key := range resp { + if key.Probe == nil { + key.Probe = &types.Probe{ProbeScheme: setting.ProtocolTCP} + } + key.PrivateKey, err = crypto.AesEncryptByKey(key.PrivateKey, aesKey.PlainText) + if err != nil { + return nil, err + } + } + return resp, nil +} + +func ListPrivateKeysInternal(log *zap.SugaredLogger) ([]*commonmodels.PrivateKey, error) { resp, err := commonrepo.NewPrivateKeyColl().List(&commonrepo.PrivateKeyArgs{}) if err != nil { + if commonrepo.IsErrNoDocuments(err) { + return []*commonmodels.PrivateKey{}, nil + } log.Errorf("PrivateKey.List error: %v", err) return resp, e.ErrListPrivateKeys } @@ -39,7 +86,7 @@ func GetPrivateKey(id string, log *zap.SugaredLogger) (*commonmodels.PrivateKey, ID: id, }) if err != nil { - log.Errorf("PrivateKey.Find %s error: %v", id, err) + log.Errorf("PrivateKey.Find %s error: %s", id, err) return resp, e.ErrGetPrivateKey } return resp, nil @@ -50,28 +97,38 @@ func CreatePrivateKey(args *commonmodels.PrivateKey, log *zap.SugaredLogger) err return e.ErrCreatePrivateKey.AddDesc("主机名称仅支持字母，数字和下划线且首个字符不以数字开头") } - if privateKeys, _ := commonrepo.NewPrivateKeyColl().List(&commonrepo.PrivateKeyArgs{Name: args.Name}); len(privateKeys) > 0 { + privateKeyArgs := &commonrepo.PrivateKeyArgs{ + Name: args.Name, + } + + if privateKeys, _ := commonrepo.NewPrivateKeyColl().List(privateKeyArgs); len(privateKeys) > 0 { return e.ErrCreatePrivateKey.AddDesc("Name already exists") } err := commonrepo.NewPrivateKeyColl().Create(args) if err != nil { - log.Errorf("PrivateKey.Create error: %v", err) + log.Errorf("failed to create privateKey, error: %s", err) return e.ErrCreatePrivateKey } return nil } func UpdatePrivateKey(id string, args *commonmodels.PrivateKey, log *zap.SugaredLogger) error { - err := commonrepo.NewPrivateKeyColl().Update(id, args) + _, err := commonrepo.NewPrivateKeyColl().Find(commonrepo.FindPrivateKeyOption{ID: id}) + if err != nil { + log.Errorf("failed to find privateKey with id: %s, error: %s", id, err) + return e.ErrUpdatePrivateKey.AddErr(fmt.Errorf("failed to find privateKey with id: %s, err: %s", id, err)) + } + + err = commonrepo.NewPrivateKeyColl().Update(id, args) if err != nil { - log.Errorf("PrivateKey.Update %s error: %v", id, err) - return e.ErrUpdatePrivateKey + log.Errorf("failed to update privateKey, error: %s", err) + return e.ErrUpdatePrivateKey.AddErr(err) } return nil } -func DeletePrivateKey(id string, log *zap.SugaredLogger) error { +func DeletePrivateKey(id, userName string, log *zap.SugaredLogger) error { // 检查该私钥是否被引用 buildOpt := &commonrepo.BuildListOption{PrivateKeyID: id} builds, err := commonrepo.NewBuildColl().List(buildOpt) @@ -82,8 +139,106 @@ func DeletePrivateKey(id string, log *zap.SugaredLogger) error { err = commonrepo.NewPrivateKeyColl().Delete(id) if err != nil { - log.Errorf("PrivateKey.Delete %s error: %v", id, err) + log.Errorf("PrivateKey.Delete %s error: %s", id, err) return e.ErrDeletePrivateKey } + // update releated services , which contains the privateKey + services, err := commonrepo.NewServiceColl().ListMaxRevisions(&commonrepo.ServiceListOption{Type: "pm"}) + if err != nil { + return err + } + for _, service := range services { + hostIDsSet := sets.NewString() + for _, config := range service.EnvConfigs { + hostIDsSet.Insert(config.HostIDs...) + } + if !hostIDsSet.Has(id) { + continue + } + // has related hostID + envConfigs := []*commonmodels.EnvConfig{} + for _, config := range service.EnvConfigs { + hostIdsSet := sets.NewString(config.HostIDs...) + if hostIdsSet.Has(id) { + hostIdsSet.Delete(id) + config.HostIDs = hostIdsSet.List() + } + envConfigs = append(envConfigs, config) + } + + envStatus, err := pm.GenerateEnvStatus(service.EnvConfigs, log) + if err != nil { + log.Errorf("GenerateEnvStatus err:%s", err) + continue + } + args := &commonservice.ServiceTmplBuildObject{ + ServiceTmplObject: &commonservice.ServiceTmplObject{ + ProductName: service.ProductName, + ServiceName: service.ServiceName, + Visibility: service.Visibility, + Revision: service.Revision, + Type: service.Type, + Username: userName, + HealthChecks: service.HealthChecks, + EnvConfigs: envConfigs, + EnvStatuses: envStatus, + From: "deletePriveteKey", + }, + Build: &commonmodels.Build{Name: service.BuildName}, + } + if err := commonservice.UpdatePmServiceTemplate(userName, args, log); err != nil { + log.Errorf("UpdatePmServiceTemplate err :%s", err) + continue + } + } + return nil +} + +func ListLabels() ([]string, error) { + return commonrepo.NewPrivateKeyColl().DistinctLabels() +} + +// override: Full coverage (temporarily reserved) +// increment: Incremental coverage +// patch: Overwrite existing +func BatchCreatePrivateKey(args []*commonmodels.PrivateKey, option, username string, log *zap.SugaredLogger) error { + switch option { + case "increment": + for _, currentPrivateKey := range args { + if !config.CVMNameRegex.MatchString(currentPrivateKey.Name) { + return e.ErrBulkCreatePrivateKey.AddDesc("主机名称仅支持字母，数字和下划线且首个字符不以数字开头") + } + + if privateKeys, _ := commonrepo.NewPrivateKeyColl().List(&commonrepo.PrivateKeyArgs{Name: currentPrivateKey.Name}); len(privateKeys) > 0 { + continue + } + + currentPrivateKey.UpdateBy = username + if err := commonrepo.NewPrivateKeyColl().Create(currentPrivateKey); err != nil { + log.Errorf("PrivateKey.Create error: %s", err) + return e.ErrBulkCreatePrivateKey.AddDesc("bulk add privateKey failed") + } + } + + case "patch": + for _, currentPrivateKey := range args { + if !config.CVMNameRegex.MatchString(currentPrivateKey.Name) { + return e.ErrBulkCreatePrivateKey.AddDesc("主机名称仅支持字母，数字和下划线且首个字符不以数字开头") + } + currentPrivateKey.UpdateBy = username + if privateKeys, _ := commonrepo.NewPrivateKeyColl().List(&commonrepo.PrivateKeyArgs{Name: currentPrivateKey.Name}); len(privateKeys) > 0 { + if err := commonrepo.NewPrivateKeyColl().Update(privateKeys[0].ID.Hex(), currentPrivateKey); err != nil { + log.Errorf("PrivateKey.update error: %s", err) + return e.ErrBulkCreatePrivateKey.AddDesc("bulk update privateKey failed") + } + continue + } + if err := commonrepo.NewPrivateKeyColl().Create(currentPrivateKey); err != nil { + log.Errorf("PrivateKey.Create error: %s", err) + return e.ErrBulkCreatePrivateKey.AddDesc("bulk add privateKey failed") + } + } + } + return nil } diff --git a/pkg/microservice/aslan/core/system/service/registry.go b/pkg/microservice/aslan/core/system/service/registry.go index 0aa8288eea2bfb3bfed246e8ec4f924d3ffd4758..32a940051a09269a79e6af63663b50af7d44247a 100644 --- a/pkg/microservice/aslan/core/system/service/registry.go +++ b/pkg/microservice/aslan/core/system/service/registry.go @@ -17,18 +17,21 @@ limitations under the License. package service import ( + "errors" "fmt" "strings" "go.uber.org/zap" + "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/registry" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" e "github.com/koderover/zadig/pkg/tool/errors" + registrytool "github.com/koderover/zadig/pkg/tool/registries" "github.com/koderover/zadig/pkg/util" ) @@ -54,7 +57,7 @@ const ( // ListRegistries 为了抹掉ak和sk的数据 func ListRegistries(log *zap.SugaredLogger) ([]*commonmodels.RegistryNamespace, error) { - registryNamespaces, err := commonrepo.NewRegistryNamespaceColl().FindAll(&commonrepo.FindRegOps{}) + registryNamespaces, err := commonservice.ListRegistryNamespaces("", false, log) if err != nil { log.Errorf("RegistryNamespace.List error: %v", err) return registryNamespaces, fmt.Errorf("RegistryNamespace.List error: %v", err) @@ -69,43 +72,21 @@ func ListRegistries(log *zap.SugaredLogger) ([]*commonmodels.RegistryNamespace, func CreateRegistryNamespace(username string, args *commonmodels.RegistryNamespace, log *zap.SugaredLogger) error { regOps := new(commonrepo.FindRegOps) regOps.IsDefault = true - registryInfoList, _ := GetRegistryNamespaces(regOps, log) + defaultReg, isSystemDefault, err := commonservice.FindDefaultRegistry(false, log) + if err != nil { + log.Warnf("failed to find the default registry, the error is: %s", err) + } if args.IsDefault { - for _, registryInfo := range registryInfoList { - registryInfo.IsDefault = false - err := UpdateRegistryNamespaceDefault(registryInfo, log) + if defaultReg != nil && !isSystemDefault { + defaultReg.IsDefault = false + err := UpdateRegistryNamespaceDefault(defaultReg, log) if err != nil { log.Errorf("updateRegistry error: %v", err) return fmt.Errorf("RegistryNamespace.Create error: %v", err) } } - - envs, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{ExcludeSource: setting.SourceFromExternal}) - if err != nil { - log.Errorf("Failed to list namespaces to update") - } - for _, env := range envs { - go func(prod *commonmodels.Product) { - kubeClient, err := kube.GetKubeClient(prod.ClusterID) - if err != nil { - log.Errorf("[updateRegistry] Failed to get kubecli for namespace: %s", prod.Namespace) - return - } - err = commonservice.EnsureDefaultRegistrySecret(prod.Namespace, kubeClient, log) - if err != nil { - log.Errorf("[updateRegistry] Failed to update registry secret for namespace: %s, the error is: %+v", prod.Namespace, err) - } - }(env) - } } else { - hasDefault := false - for _, registryInfo := range registryInfoList { - if registryInfo.IsDefault { - hasDefault = true - break - } - } - if !hasDefault { + if isSystemDefault { log.Errorf("create registry error: There must be at least 1 default registry") return fmt.Errorf("RegistryNamespace.Create error: %s", "There must be at least 1 default registry") } @@ -118,32 +99,29 @@ func CreateRegistryNamespace(username string, args *commonmodels.RegistryNamespa log.Errorf("RegistryNamespace.Create error: %v", err) return fmt.Errorf("RegistryNamespace.Create error: %v", err) } - return nil + + return SyncDinDForRegistries() } func UpdateRegistryNamespace(username, id string, args *commonmodels.RegistryNamespace, log *zap.SugaredLogger) error { regOps := new(commonrepo.FindRegOps) regOps.IsDefault = true - registryInfoList, _ := GetRegistryNamespaces(regOps, log) + defaultReg, isSystemDefault, err := commonservice.FindDefaultRegistry(false, log) + if err != nil { + log.Warnf("failed to find the default registry, the error is: %s", err) + } if args.IsDefault { - for _, registryInfo := range registryInfoList { - registryInfo.IsDefault = false - err := UpdateRegistryNamespaceDefault(registryInfo, log) + if defaultReg != nil && !isSystemDefault { + defaultReg.IsDefault = false + err := UpdateRegistryNamespaceDefault(defaultReg, log) if err != nil { log.Errorf("updateRegistry error: %v", err) return fmt.Errorf("RegistryNamespace.Update error: %v", err) } } } else { - hasDefault := false - for _, registryInfo := range registryInfoList { - if registryInfo.ID != args.ID { - hasDefault = true - break - } - } - if !hasDefault { - log.Errorf("update registry error: There must be at least 1 default registry") + if isSystemDefault || id == defaultReg.ID.Hex() { + log.Errorf("create registry error: There must be at least 1 default registry") return fmt.Errorf("RegistryNamespace.Create error: %s", "There must be at least 1 default registry") } } @@ -154,38 +132,56 @@ func UpdateRegistryNamespace(username, id string, args *commonmodels.RegistryNam log.Errorf("RegistryNamespace.Update error: %v", err) return fmt.Errorf("RegistryNamespace.Update error: %v", err) } + return SyncDinDForRegistries() +} - envs, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{ExcludeSource: setting.SourceFromExternal}) +func DeleteRegistryNamespace(id string, log *zap.SugaredLogger) error { + registries, err := commonrepo.NewRegistryNamespaceColl().FindAll(&commonrepo.FindRegOps{}) if err != nil { - log.Errorf("Failed to list namespaces to update") + log.Errorf("RegistryNamespace.FindAll error: %s", err) + return err } + var ( + isDefault = false + registryNamespaces []*commonmodels.RegistryNamespace + ) + envs, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{ + ExcludeStatus: []string{setting.ProductStatusDeleting}, + }) + for _, env := range envs { - go func(prod *commonmodels.Product) { - kubeClient, err := kube.GetKubeClient(prod.ClusterID) - if err != nil { - log.Errorf("[updateRegistry] Failed to get kubecli for namespace: %s", prod.Namespace) - return - } - err = commonservice.EnsureDefaultRegistrySecret(prod.Namespace, kubeClient, log) - if err != nil { - log.Errorf("[updateRegistry] Failed to update registry secret for namespace: %s, the error is: %+v", prod.Namespace, err) - } - }(env) + if env.RegistryID == id { + return errors.New("The registry cannot be deleted, it's being used by environment") + } + } + + // whether it is the default registry + for _, registry := range registries { + if registry.ID.Hex() == id && registry.IsDefault { + isDefault = true + continue + } + registryNamespaces = append(registryNamespaces, registry) } - return nil -} -func DeleteRegistryNamespace(id string, log *zap.SugaredLogger) error { if err := commonrepo.NewRegistryNamespaceColl().Delete(id); err != nil { - log.Errorf("RegistryNamespace.Delete error: %v", err) + log.Errorf("RegistryNamespace.Delete error: %s", err) return err } - return nil + + if isDefault && len(registryNamespaces) > 0 { + registryNamespaces[0].IsDefault = true + if err := commonrepo.NewRegistryNamespaceColl().Update(registryNamespaces[0].ID.Hex(), registryNamespaces[0]); err != nil { + log.Errorf("RegistryNamespace.Update error: %s", err) + return err + } + } + return SyncDinDForRegistries() } func ListAllRepos(log *zap.SugaredLogger) ([]*RepoInfo, error) { repoInfos := make([]*RepoInfo, 0) - resp, err := commonrepo.NewRegistryNamespaceColl().FindAll(&commonrepo.FindRegOps{}) + resp, err := commonservice.ListRegistryNamespaces("", false, log) if err != nil { log.Errorf("RegistryNamespace.List error: %v", err) return nil, fmt.Errorf("RegistryNamespace.List error: %v", err) @@ -203,17 +199,14 @@ func ListAllRepos(log *zap.SugaredLogger) ([]*RepoInfo, error) { return repoInfos, nil } -func GetRegistryNamespace(regOps *commonrepo.FindRegOps, log *zap.SugaredLogger) (*commonmodels.RegistryNamespace, error) { - resp, err := commonrepo.NewRegistryNamespaceColl().Find(regOps) - if err != nil { - log.Errorf("RegistryNamespace.get error: %v", err) - return nil, fmt.Errorf("RegistryNamespace.get error: %v", err) - } - return resp, nil -} - func ListReposTags(registryInfo *commonmodels.RegistryNamespace, names []string, logger *zap.SugaredLogger) ([]*RepoImgResp, error) { - repos, err := registry.NewV2Service(registryInfo.RegProvider).ListRepoImages(registry.ListRepoImagesOption{ + var regService registry.Service + if registryInfo.AdvancedSetting != nil { + regService = registry.NewV2Service(registryInfo.RegProvider, registryInfo.AdvancedSetting.TLSEnabled, registryInfo.AdvancedSetting.TLSCert) + } else { + regService = registry.NewV2Service(registryInfo.RegProvider, true, "") + } + repos, err := regService.ListRepoImages(registry.ListRepoImagesOption{ Endpoint: registry.Endpoint{ Addr: registryInfo.RegAddr, Ak: registryInfo.AccessKey, @@ -229,7 +222,7 @@ func ListReposTags(registryInfo *commonmodels.RegistryNamespace, names []string, for _, repo := range repos.Repos { for _, tag := range repo.Tags { img := &RepoImgResp{ - Host: util.GetURLHostName(registryInfo.RegAddr), + Host: util.TrimURLScheme(registryInfo.RegAddr), Owner: repo.Namespace, Name: repo.Name, Tag: tag, @@ -246,7 +239,13 @@ func ListReposTags(registryInfo *commonmodels.RegistryNamespace, names []string, func GetRepoTags(registryInfo *commonmodels.RegistryNamespace, name string, log *zap.SugaredLogger) (*registry.ImagesResp, error) { var resp *registry.ImagesResp - repos, err := registry.NewV2Service(registryInfo.RegProvider).ListRepoImages(registry.ListRepoImagesOption{ + var regService registry.Service + if registryInfo.AdvancedSetting != nil { + regService = registry.NewV2Service(registryInfo.RegProvider, registryInfo.AdvancedSetting.TLSEnabled, registryInfo.AdvancedSetting.TLSCert) + } else { + regService = registry.NewV2Service(registryInfo.RegProvider, true, "") + } + repos, err := regService.ListRepoImages(registry.ListRepoImagesOption{ Endpoint: registry.Endpoint{ Addr: registryInfo.RegAddr, Ak: registryInfo.AccessKey, @@ -281,15 +280,6 @@ func GetRepoTags(registryInfo *commonmodels.RegistryNamespace, name string, log return resp, err } -func GetRegistryNamespaces(regOps *commonrepo.FindRegOps, log *zap.SugaredLogger) ([]*commonmodels.RegistryNamespace, error) { - resp, err := commonrepo.NewRegistryNamespaceColl().FindAll(regOps) - if err != nil { - log.Errorf("RegistryNamespace.findAll error: %+v", err) - return nil, fmt.Errorf("RegistryNamespace.findAll error: %v", err) - } - return resp, nil -} - func UpdateRegistryNamespaceDefault(args *commonmodels.RegistryNamespace, log *zap.SugaredLogger) error { if err := commonrepo.NewRegistryNamespaceColl().Update(args.ID.Hex(), args); err != nil { log.Errorf("UpdateRegistryNamespaceDefault.Update error: %v", err) @@ -297,3 +287,32 @@ func UpdateRegistryNamespaceDefault(args *commonmodels.RegistryNamespace, log *z } return nil } + +func SyncDinDForRegistries() error { + registries, err := commonrepo.NewRegistryNamespaceColl().FindAll(&commonrepo.FindRegOps{}) + if err != nil { + return fmt.Errorf("failed to list registry to update dind, err: %s", err) + } + + regList := make([]*registrytool.RegistryInfoForDinDUpdate, 0) + for _, reg := range registries { + regItem := ®istrytool.RegistryInfoForDinDUpdate{ + ID: reg.ID, + RegAddr: reg.RegAddr, + } + if reg.AdvancedSetting != nil { + regItem.AdvancedSetting = ®istrytool.RegistryAdvancedSetting{ + TLSEnabled: reg.AdvancedSetting.TLSEnabled, + TLSCert: reg.AdvancedSetting.TLSCert, + } + } + regList = append(regList, regItem) + } + + dynamicClient, err := kubeclient.GetDynamicKubeClient(config.HubServerAddress(), setting.LocalClusterID) + if err != nil { + return fmt.Errorf("failed to get dynamic client to update dind, err: %s", err) + } + + return registrytool.PrepareDinD(dynamicClient, config.Namespace(), regList) +} diff --git a/pkg/microservice/aslan/core/system/service/s3.go b/pkg/microservice/aslan/core/system/service/s3.go index 89f26377b4b541bfd1175baa46a02d7d454144bf..cccd90f93e731a90b00e7e75e565de3c0689f32e 100644 --- a/pkg/microservice/aslan/core/system/service/s3.go +++ b/pkg/microservice/aslan/core/system/service/s3.go @@ -17,12 +17,20 @@ limitations under the License. package service import ( + "strconv" + "strings" + "sync" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/util/wait" + "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/crypto" "github.com/koderover/zadig/pkg/tool/errors" s3tool "github.com/koderover/zadig/pkg/tool/s3" ) @@ -67,12 +75,23 @@ func CreateS3Storage(updateBy string, storage *commonmodels.S3Storage, logger *z return commonrepo.NewS3StorageColl().Create(storage) } -func ListS3Storage(logger *zap.SugaredLogger) ([]*commonmodels.S3Storage, error) { +func ListS3Storage(encryptedKey string, logger *zap.SugaredLogger) ([]*commonmodels.S3Storage, error) { stores, err := commonrepo.NewS3StorageColl().FindAll() if err == nil && len(stores) == 0 { stores = make([]*commonmodels.S3Storage, 0) } - + aesKey, err := service.GetAesKeyFromEncryptedKey(encryptedKey, logger) + if err != nil { + logger.Errorf("ListS3Storage GetAesKeyFromEncryptedKey err:%s", err) + return nil, err + } + for _, store := range stores { + store.Sk, err = crypto.AesEncryptByKey(store.Sk, aesKey.PlainText) + if err != nil { + logger.Errorf("ListS3Storage AesEncryptByKey err:%s", err) + return nil, err + } + } return stores, err } @@ -95,3 +114,81 @@ func GetS3Storage(id string, logger *zap.SugaredLogger) (*commonmodels.S3Storage return store, nil } + +func ListTars(id, kind string, serviceNames []string, logger *zap.SugaredLogger) ([]*commonmodels.TarInfo, error) { + var ( + wg wait.Group + mutex sync.RWMutex + tarInfos = make([]*commonmodels.TarInfo, 0) + store *commonmodels.S3Storage + defaultS3 s3.S3 + defaultURL string + err error + ) + + store, err = commonrepo.NewS3StorageColl().Find(id) + if err != nil { + logger.Errorf("can't find store by id:%s err:%s", id, err) + return nil, err + } + defaultS3 = s3.S3{ + S3Storage: store, + } + defaultURL, err = defaultS3.GetEncryptedURL() + if err != nil { + logger.Errorf("defaultS3 GetEncryptedURL err:%s", err) + return nil, err + } + + for _, serviceName := range serviceNames { + // Change the service name to underscore splicing + newServiceName := serviceName + wg.Start(func() { + deliveryArtifactArgs := &commonrepo.DeliveryArtifactArgs{ + Name: newServiceName, + Type: kind, + Source: string(config.WorkflowType), + PackageStorageURI: store.Endpoint + "/" + store.Bucket, + } + deliveryArtifacts, err := commonrepo.NewDeliveryArtifactColl().ListTars(deliveryArtifactArgs) + if err != nil { + logger.Errorf("ListTars err:%s", err) + return + } + deliveryArtifactArgs.Name = newServiceName + "_" + newServiceName + newDeliveryArtifacts, err := commonrepo.NewDeliveryArtifactColl().ListTars(deliveryArtifactArgs) + if err != nil { + logger.Errorf("ListTars err:%s", err) + return + } + deliveryArtifacts = append(deliveryArtifacts, newDeliveryArtifacts...) + for _, deliveryArtifact := range deliveryArtifacts { + activities, _, err := commonrepo.NewDeliveryActivityColl().List(&commonrepo.DeliveryActivityArgs{ArtifactID: deliveryArtifact.ID.Hex()}) + if err != nil { + logger.Errorf("deliveryActivity.list err:%s", err) + return + } + urlArr := strings.Split(activities[0].URL, "/") + workflowName := urlArr[len(urlArr)-2] + taskIDStr := urlArr[len(urlArr)-1] + taskID, err := strconv.Atoi(taskIDStr) + if err != nil { + logger.Errorf("string convert to int err:%s", err) + return + } + + mutex.Lock() + tarInfos = append(tarInfos, &commonmodels.TarInfo{ + URL: defaultURL, + Name: newServiceName, + FileName: deliveryArtifact.Image, + WorkflowName: workflowName, + TaskID: int64(taskID), + }) + mutex.Unlock() + } + }) + } + wg.Wait() + return tarInfos, nil +} diff --git a/pkg/microservice/aslan/core/system/service/sonar.go b/pkg/microservice/aslan/core/system/service/sonar.go new file mode 100644 index 0000000000000000000000000000000000000000..a3aae4a0fa5bd88813565ad20392adfa542cdae3 --- /dev/null +++ b/pkg/microservice/aslan/core/system/service/sonar.go @@ -0,0 +1,140 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + + "go.uber.org/zap" + + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +func CreateSonarIntegration(args *SonarIntegration, log *zap.SugaredLogger) error { + err := commonrepo.NewSonarIntegrationColl().Create(context.TODO(), &commonmodels.SonarIntegration{ + ServerAddress: args.ServerAddress, + Token: args.Token, + }) + if err != nil { + log.Errorf("Create external system error: %s", err) + return e.ErrCreateExternalLink.AddErr(err) + } + return nil +} + +func UpdateSonarIntegration(id string, integration *SonarIntegration, log *zap.SugaredLogger) error { + err := commonrepo.NewSonarIntegrationColl().Update( + context.TODO(), + id, + &commonmodels.SonarIntegration{ + ServerAddress: integration.ServerAddress, + Token: integration.Token, + }, + ) + if err != nil { + log.Errorf("update external system error: %s", err) + } + return err +} + +func ListSonarIntegration(log *zap.SugaredLogger) ([]*SonarIntegration, int64, error) { + // for now paging is not supported + sonarList, length, err := commonrepo.NewSonarIntegrationColl().List(context.TODO(), 0, 0) + if err != nil { + log.Errorf("Failed to list sonar integration from db, the error is: %s", err) + return nil, 0, err + } + resp := make([]*SonarIntegration, 0) + for _, sonar := range sonarList { + resp = append(resp, &SonarIntegration{ + ID: sonar.ID.Hex(), + ServerAddress: sonar.ServerAddress, + Token: sonar.Token, + }) + } + return resp, length, nil +} + +func GetSonarIntegration(id string, log *zap.SugaredLogger) (*SonarIntegration, error) { + resp := new(SonarIntegration) + sonarIntegration, err := commonrepo.NewSonarIntegrationColl().GetByID(context.TODO(), id) + if err != nil { + log.Errorf("Failed to get sonar integration detail from id %s, the error is: %s", id, err) + return nil, err + } + resp.ID = sonarIntegration.ID.Hex() + resp.ServerAddress = sonarIntegration.ServerAddress + resp.Token = sonarIntegration.Token + return resp, nil +} + +func DeleteSonarIntegration(id string, log *zap.SugaredLogger) error { + err := commonrepo.NewSonarIntegrationColl().DeleteByID(context.TODO(), id) + if err != nil { + log.Errorf("Failed to delete sonar integration of id: %s, the error is: %s", id, err) + } + return err +} + +type sonarValidationResponse struct { + Valid bool `json:"valid"` +} + +func ValidateSonarIntegration(arg *SonarIntegration, log *zap.SugaredLogger) error { + validateAPI := fmt.Sprintf("%s/%s", arg.ServerAddress, "api/authentication/validate") + + req, err := http.NewRequest("GET", validateAPI, nil) + if err != nil { + log.Errorf("failed to create http request to API: %s with error: %s", validateAPI, err) + return err + } + + req.SetBasicAuth(arg.Token, "") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + log.Errorf("failed to request the sonar validate API, the error is: %s", err) + return err + } + defer resp.Body.Close() + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + log.Errorf("failed to read response body, the error is: %s", err) + return err + } + if resp.StatusCode == http.StatusOK { + res := &sonarValidationResponse{} + err = json.Unmarshal(bodyBytes, res) + if err != nil { + log.Errorf("failed to ready response body, the error is: %s", err) + return err + } + if res.Valid { + return nil + } + return errors.New("the token validation failed.") + } + + return fmt.Errorf("the server responded with code: %d and message: %s", resp.StatusCode, string(bodyBytes)) +} diff --git a/pkg/microservice/aslan/core/system/service/system_cache.go b/pkg/microservice/aslan/core/system/service/system_cache.go index ca4d9ce5202fdf3f515c5f682b8c7432e2ad9b28..ffa3646a39694da3b25501c02b8727757b384df9 100644 --- a/pkg/microservice/aslan/core/system/service/system_cache.go +++ b/pkg/microservice/aslan/core/system/service/system_cache.go @@ -17,25 +17,31 @@ limitations under the License. package service import ( + "context" + "fmt" "sort" "strings" "sync" "time" "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" "github.com/koderover/zadig/pkg/shared/kube/wrapper" e "github.com/koderover/zadig/pkg/tool/errors" - kubetool "github.com/koderover/zadig/pkg/tool/kube/client" "github.com/koderover/zadig/pkg/tool/kube/getter" "github.com/koderover/zadig/pkg/tool/kube/podexec" + "github.com/koderover/zadig/pkg/types" ) +var cleanCacheLock sync.Mutex + const ( CleanStatusUnStart = "unStart" CleanStatusSuccess = "success" @@ -43,17 +49,50 @@ const ( CleanStatusFailed = "failed" ) -// CleanImageCache 清理镜像缓存 +//SetCron set the docker clean cron +func SetCron(cron string, cronEnabled bool, logger *zap.SugaredLogger) error { + dindCleans, err := commonrepo.NewDindCleanColl().List() + if err != nil { + logger.Errorf("list dind cleans err:%s", err) + return err + } + switch len(dindCleans) { + case 0: + dindClean := &commonmodels.DindClean{ + Status: CleanStatusUnStart, + DindCleanInfos: []*commonmodels.DindCleanInfo{}, + Cron: cron, + CronEnabled: cronEnabled, + } + + if err := commonrepo.NewDindCleanColl().Upsert(dindClean); err != nil { + return e.ErrCreateDindClean.AddErr(err) + } + case 1: + dindClean := dindCleans[0] + dindClean.Status = CleanStatusSuccess + dindClean.DindCleanInfos = dindCleans[0].DindCleanInfos + dindClean.Cron = cron + dindClean.CronEnabled = cronEnabled + if err := commonrepo.NewDindCleanColl().Upsert(dindClean); err != nil { + return e.ErrUpdateDindClean.AddErr(err) + } + } + return nil +} + func CleanImageCache(logger *zap.SugaredLogger) error { - //Get pod list by label and namespace - //判断当前的状态，cleaning状态下不做清理 - var mu sync.Mutex - mu.Lock() - defer func() { - mu.Unlock() - }() + // TODO: We should return immediately instead of waiting for the lock if a cleanup task is performed. + // Since `golang-1.18`, `sync.Mutex` provides a `TryLock()` method. For now, we can continue with the previous + // logic and replace it with `TryLock()` after upgrading to `golang-1.18+` to return immediately. + cleanCacheLock.Lock() + defer cleanCacheLock.Unlock() + + dindCleans, err := commonrepo.NewDindCleanColl().List() + if err != nil { + return fmt.Errorf("failed to list data in `dind_clean` table: %s", err) + } - dindCleans, _ := commonrepo.NewDindCleanColl().List() switch len(dindCleans) { case 0: dindClean := &commonmodels.DindClean{ @@ -71,66 +110,91 @@ func CleanImageCache(logger *zap.SugaredLogger) error { } dindClean.Status = CleanStatusCleaning dindClean.DindCleanInfos = []*commonmodels.DindCleanInfo{} + dindClean.Cron = dindCleans[0].Cron + dindClean.CronEnabled = dindCleans[0].CronEnabled if err := commonrepo.NewDindCleanColl().Upsert(dindClean); err != nil { return e.ErrUpdateDindClean.AddErr(err) } } - go func() { + dindPods, err := getDindPods() + if err != nil { + logger.Errorf("Failed to list dind pods: %s", err) + return commonrepo.NewDindCleanColl().Upsert(&commonmodels.DindClean{ + Status: CleanStatusFailed, + DindCleanInfos: []*commonmodels.DindCleanInfo{}, + Cron: dindCleans[0].Cron, + CronEnabled: dindCleans[0].CronEnabled, + }) + } + logger.Infof("Total dind Pods to be cleaned up: %d", len(dindPods)) + + // Note: Since the total number of dind instances of Zadig users will not exceed `50` within one or two years + // (at this time, the resource amount may be `200C400GiB`, and the resource cost is too high), concurrency can be + // left out of consideration. + timeout, cancel := context.WithTimeout(context.Background(), 20*time.Minute) + defer cancel() + res := make(chan *commonmodels.DindClean) + go func(ch chan *commonmodels.DindClean) { var ( - namespace = config.Namespace() - status = CleanStatusSuccess + status = CleanStatusSuccess ) - selector := labels.Set{setting.ComponentLabel: "dind"}.AsSelector() - pods, err := getter.ListPods(namespace, selector, kubetool.Client()) - - if err != nil { - logger.Errorf("[%s]list dind pods error: %v", namespace, err) - commonrepo.NewDindCleanColl().Upsert(&commonmodels.DindClean{ - Status: CleanStatusFailed, - DindCleanInfos: []*commonmodels.DindCleanInfo{}, - }) - return - } - - dindInfos := make([]*commonmodels.DindCleanInfo, 0, len(pods)) + dindInfos := make([]*commonmodels.DindCleanInfo, 0, len(dindPods)) var wg sync.WaitGroup - for _, pod := range pods { - if wrapper.Pod(pod).Ready() { - logger.Infof("[%s] pod [%s] dind cache cleaning", namespace, pod.Name) - wg.Add(1) - go func(podName string) { - defer wg.Done() - dindInfo := &commonmodels.DindCleanInfo{ - PodName: podName, - StartTime: time.Now().Unix(), - } - output, err := dockerPrune(namespace, podName, logger) - if err != nil { - logger.Errorf("[%s] pod [%s] dind cache clean error: %v", namespace, podName, err) - dindInfo.ErrorMessage = err.Error() - } - logger.Infof("[%s] pod [%s] dind cache finish", namespace, podName) - dindInfo.EndTime = time.Now().Unix() - dindInfo.CleanInfo = output - dindInfos = append(dindInfos, dindInfo) - }(pod.Name) + for _, dindPod := range dindPods { + if !wrapper.Pod(dindPod.Pod).Ready() { + continue } + + logger.Infof("Begin to clean up cache of dind %q in ns %q of cluster %q.", dindPod.Pod.Name, dindPod.Pod.Namespace, dindPod.ClusterID) + wg.Add(1) + go func(podInfo types.DindPod) { + defer wg.Done() + dindInfo := &commonmodels.DindCleanInfo{ + PodName: fmt.Sprintf("%s:%s", podInfo.ClusterName, podInfo.Pod.Name), + StartTime: time.Now().Unix(), + } + output, err := dockerPrune(podInfo.ClusterID, podInfo.Pod.Namespace, podInfo.Pod.Name, logger) + if err != nil { + logger.Warnf("Failed to clean up cache of dind %q in ns %q of cluster %q: %s", podInfo.Pod.Name, podInfo.Pod.Namespace, podInfo.ClusterID, err) + dindInfo.ErrorMessage = err.Error() + } + logger.Infof("Finish cleaning up cache of dind %q in ns %q of cluster %q.", podInfo.Pod.Name, podInfo.Pod.Namespace, podInfo.ClusterID) + + dindInfo.EndTime = time.Now().Unix() + dindInfo.CleanInfo = output + dindInfos = append(dindInfos, dindInfo) + }(dindPod) } - wg.Wait() + wg.Wait() for _, dindInfo := range dindInfos { if dindInfo.ErrorMessage != "" { status = CleanStatusFailed break } } - commonrepo.NewDindCleanColl().Upsert(&commonmodels.DindClean{ + res <- &commonmodels.DindClean{ Status: status, DindCleanInfos: dindInfos, + Cron: dindCleans[0].Cron, + CronEnabled: dindCleans[0].CronEnabled, + } + + }(res) + + select { + case <-timeout.Done(): + commonrepo.NewDindCleanColl().Upsert(&commonmodels.DindClean{ + Status: CleanStatusFailed, + DindCleanInfos: []*commonmodels.DindCleanInfo{}, + Cron: dindCleans[0].Cron, + CronEnabled: dindCleans[0].CronEnabled, }) - }() + case info := <-res: + commonrepo.NewDindCleanColl().Upsert(info) + } return nil } @@ -157,19 +221,30 @@ func GetOrCreateCleanCacheState() (*commonmodels.DindClean, error) { Status: dindCleans[0].Status, UpdateTime: dindCleans[0].UpdateTime, DindCleanInfos: dindCleans[0].DindCleanInfos, + Cron: dindCleans[0].Cron, + CronEnabled: dindCleans[0].CronEnabled, } return dindClean, nil } -func dockerPrune(namespace, podName string, logger *zap.SugaredLogger) (string, error) { - cleanInfo, errString, _, err := podexec.ExecWithOptions(podexec.ExecOptions{ +func dockerPrune(clusterID, namespace, podName string, logger *zap.SugaredLogger) (string, error) { + kclient, err := kubeclient.GetClientset(config.HubServerAddress(), clusterID) + if err != nil { + return "", fmt.Errorf("failed to get clientset for cluster %q: %s", clusterID, err) + } + + restConfig, err := kubeclient.GetRESTConfig(config.HubServerAddress(), clusterID) + if err != nil { + return "", fmt.Errorf("failed to get rest config for cluster %q: %s", clusterID, err) + } + + cleanInfo, errString, _, err := podexec.KubeExec(kclient, restConfig, podexec.ExecOptions{ Command: []string{"docker", "system", "prune", "--volumes", "-a", "-f"}, Namespace: namespace, PodName: podName, }) - if err != nil { - logger.Infof("Failed to execute docker prune: %s, err: %s", errString, err) + logger.Errorf("Failed to execute docker prune: %s, err: %s", errString, err) cleanInfo = errString } @@ -181,3 +256,48 @@ func dockerPrune(namespace, podName string, logger *zap.SugaredLogger) (string, return cleanInfo, err } + +func getDindPods() ([]types.DindPod, error) { + activeClusters, err := commonrepo.NewK8SClusterColl().FindActiveClusters() + if err != nil { + return nil, fmt.Errorf("failed to get active cluster: %s", err) + } + + dindPods := []types.DindPod{} + for _, cluster := range activeClusters { + clusterID := cluster.ID.Hex() + + var ns string + switch clusterID { + case setting.LocalClusterID: + ns = config.Namespace() + default: + ns = setting.AttachedClusterNamespace + } + + pods, err := getDindPodsInCluster(clusterID, ns) + if err != nil { + return nil, fmt.Errorf("failed to get dind pods in ns %q of cluster %q: %s", ns, clusterID, err) + } + + for _, pod := range pods { + dindPods = append(dindPods, types.DindPod{ + ClusterID: clusterID, + ClusterName: cluster.Name, + Pod: pod, + }) + } + } + + return dindPods, nil +} + +func getDindPodsInCluster(clusterID, ns string) ([]*corev1.Pod, error) { + kclient, err := kubeclient.GetKubeClient(config.HubServerAddress(), clusterID) + if err != nil { + return nil, fmt.Errorf("failed to get kube client for cluster %q: %s", clusterID, err) + } + + dindSelector := labels.Set{setting.ComponentLabel: "dind"}.AsSelector() + return getter.ListPods(ns, dindSelector, kclient) +} diff --git a/pkg/microservice/aslan/core/system/service/types.go b/pkg/microservice/aslan/core/system/service/types.go new file mode 100644 index 0000000000000000000000000000000000000000..f5a170535960b1d17fd1318722761fff33b390da --- /dev/null +++ b/pkg/microservice/aslan/core/system/service/types.go @@ -0,0 +1,35 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +type ExternalSystemDetail struct { + ID string `json:"id"` + Name string `json:"name"` + Server string `json:"server"` + APIToken string `json:"api_token,omitempty"` +} + +type WorkflowConcurrencySettings struct { + WorkflowConcurrency int64 `json:"workflow_concurrency"` + BuildConcurrency int64 `json:"build_concurrency"` +} + +type SonarIntegration struct { + ID string `json:"id"` + ServerAddress string `json:"server_address"` + Token string `json:"token"` +} diff --git a/pkg/microservice/aslan/core/templatestore/handler/build.go b/pkg/microservice/aslan/core/templatestore/handler/build.go new file mode 100644 index 0000000000000000000000000000000000000000..604663e1b4ed63c9319faad3e2508a8bfd83ee62 --- /dev/null +++ b/pkg/microservice/aslan/core/templatestore/handler/build.go @@ -0,0 +1,91 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "encoding/json" + + "github.com/gin-gonic/gin" + + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + templateservice "github.com/koderover/zadig/pkg/microservice/aslan/core/templatestore/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +func GetBuildTemplate(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = templateservice.GetBuildTemplateByID(c.Param("id")) +} + +func ListBuildTemplates(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := &listYamlQuery{} + if err := c.ShouldBindQuery(args); err != nil { + ctx.Err = err + return + } + + ctx.Resp, ctx.Err = templateservice.ListBuildTemplates(args.PageNum, args.PageSize) +} + +func AddBuildTemplate(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(commonmodels.BuildTemplate) + err := c.BindJSON(args) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid Build args") + return + } + + bs, _ := json.Marshal(args) + internalhandler.InsertOperationLog(c, ctx.UserName, "", "新增", "模板-构建", args.Name, string(bs), ctx.Logger) + + ctx.Err = templateservice.AddBuildTemplate(ctx.UserName, args, ctx.Logger) +} + +func UpdateBuildTemplate(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(commonmodels.BuildTemplate) + err := c.BindJSON(args) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid Build args") + return + } + + bs, _ := json.Marshal(args) + internalhandler.InsertOperationLog(c, ctx.UserName, "", "更新", "模板-构建", args.Name, string(bs), ctx.Logger) + + ctx.Err = templateservice.UpdateBuildTemplate(c.Param("id"), args, ctx.Logger) +} + +func RemoveBuildTemplate(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + internalhandler.InsertOperationLog(c, ctx.UserName, "", "删除", "模板-构建", c.Param("id"), "", ctx.Logger) + + ctx.Err = templateservice.RemoveBuildTemplate(c.Param("id"), ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/templatestore/handler/chart.go b/pkg/microservice/aslan/core/templatestore/handler/chart.go index fd17107dc2f9c84257f5f69bcdae7a9f98a1255a..4e77f195170c9acdd329edc1fa45daf45e1e4106 100644 --- a/pkg/microservice/aslan/core/templatestore/handler/chart.go +++ b/pkg/microservice/aslan/core/templatestore/handler/chart.go @@ -17,11 +17,15 @@ limitations under the License. package handler import ( + "encoding/json" + "github.com/gin-gonic/gin" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/fs" - "github.com/koderover/zadig/pkg/microservice/aslan/core/templatestore/service" + templateservice "github.com/koderover/zadig/pkg/microservice/aslan/core/templatestore/service" internalhandler "github.com/koderover/zadig/pkg/shared/handler" + "github.com/koderover/zadig/pkg/tool/errors" ) type addChartArgs struct { @@ -34,7 +38,14 @@ func GetChartTemplate(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = service.GetChartTemplate(c.Param("name"), ctx.Logger) + ctx.Resp, ctx.Err = templateservice.GetChartTemplate(c.Param("name"), ctx.Logger) +} + +func GetTemplateVariables(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = templateservice.GetChartTemplateVariables(c.Param("name"), ctx.Logger) } func ListFiles(c *gin.Context) { @@ -42,14 +53,14 @@ func ListFiles(c *gin.Context) { defer func() { internalhandler.JSONResponse(c, ctx) }() // TODO: support to return multiple files in a bulk - ctx.Resp, ctx.Err = service.GetFileContent(c.Param("name"), c.Query("filePath"), c.Query("fileName"), ctx.Logger) + ctx.Resp, ctx.Err = templateservice.GetFileContentForTemplate(c.Param("name"), c.Query("filePath"), c.Query("fileName"), ctx.Logger) } func ListChartTemplates(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = service.ListChartTemplates(ctx.Logger) + ctx.Resp, ctx.Err = templateservice.ListChartTemplates(ctx.Logger) } func AddChartTemplate(c *gin.Context) { @@ -62,7 +73,10 @@ func AddChartTemplate(c *gin.Context) { return } - ctx.Err = service.AddChartTemplate(args.Name, args.DownloadFromSourceArgs, ctx.Logger) + bs, _ := json.Marshal(args) + internalhandler.InsertOperationLog(c, ctx.UserName, "", "新建", "模板库-Chart", args.Name, string(bs), ctx.Logger) + + ctx.Err = templateservice.AddChartTemplate(args.Name, args.DownloadFromSourceArgs, ctx.Logger) } func UpdateChartTemplate(c *gin.Context) { @@ -75,12 +89,39 @@ func UpdateChartTemplate(c *gin.Context) { return } - ctx.Err = service.UpdateChartTemplate(c.Param("name"), args.DownloadFromSourceArgs, ctx.Logger) + bs, _ := json.Marshal(args) + internalhandler.InsertOperationLog(c, ctx.UserName, "", "更新", "模板库-Chart", args.Name, string(bs), ctx.Logger) + + ctx.Err = templateservice.UpdateChartTemplate(c.Param("name"), args.DownloadFromSourceArgs, ctx.Logger) +} + +func UpdateChartTemplateVariables(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := make([]*commonmodels.Variable, 0) + if err := c.ShouldBindJSON(&args); err != nil { + ctx.Err = errors.ErrInvalidParam.AddErr(err) + return + } + + ctx.Err = templateservice.UpdateChartTemplateVariables(c.Param("name"), args, ctx.Logger) +} + +func SyncChartTemplateReference(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + internalhandler.InsertOperationLog(c, ctx.UserName, "", "同步", "模板库-Chart", c.Param("name"), "", ctx.Logger) + + ctx.Err = templateservice.SyncHelmTemplateReference(ctx.UserName, c.Param("name"), ctx.Logger) } func RemoveChartTemplate(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Err = service.RemoveChartTemplate(c.Param("name"), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, "", "删除", "模板库-Chart", c.Param("name"), "", ctx.Logger) + + ctx.Err = templateservice.RemoveChartTemplate(c.Param("name"), ctx.Logger) } diff --git a/pkg/microservice/aslan/core/templatestore/handler/dockerfile.go b/pkg/microservice/aslan/core/templatestore/handler/dockerfile.go new file mode 100644 index 0000000000000000000000000000000000000000..4d1cd3c161dea1f6f71dac62eea75b684dcb7bc6 --- /dev/null +++ b/pkg/microservice/aslan/core/templatestore/handler/dockerfile.go @@ -0,0 +1,154 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "encoding/json" + "errors" + + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/template" + templateservice "github.com/koderover/zadig/pkg/microservice/aslan/core/templatestore/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" +) + +func CreateDockerfileTemplate(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + req := &template.DockerfileTemplate{} + + if err := c.ShouldBindJSON(req); err != nil { + ctx.Err = err + return + } + + // some dockerfile validation stuff + err := templateservice.ValidateDockerfileTemplate(req.Content, ctx.Logger) + if err != nil { + ctx.Err = errors.New("invalid dockerfile, please check") + return + } + + bs, _ := json.Marshal(req) + internalhandler.InsertOperationLog(c, ctx.UserName, "", "新建", "模板库-Dockerfile", req.Name, string(bs), ctx.Logger) + + ctx.Err = templateservice.CreateDockerfileTemplate(req, ctx.Logger) +} + +func UpdateDockerfileTemplate(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + req := &template.DockerfileTemplate{} + + if err := c.ShouldBindJSON(req); err != nil { + ctx.Err = err + return + } + + // some dockerfile validation stuff + err := templateservice.ValidateDockerfileTemplate(req.Content, ctx.Logger) + if err != nil { + ctx.Err = errors.New("invalid dockerfile, please check") + return + } + + bs, _ := json.Marshal(req) + internalhandler.InsertOperationLog(c, ctx.UserName, "", "更新", "模板库-Dockerfile", req.Name, string(bs), ctx.Logger) + + ctx.Err = templateservice.UpdateDockerfileTemplate(c.Param("id"), req, ctx.Logger) +} + +type listDockerfileQuery struct { + PageSize int `json:"page_size" form:"page_size,default=100"` + PageNum int `json:"page_num" form:"page_num,default=1"` +} + +type ListDockefileResp struct { + DockerfileTemplates []*template.DockerfileListObject `json:"dockerfile_template"` + Total int `json:"total"` +} + +func ListDockerfileTemplate(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + // Query Verification + args := listDockerfileQuery{} + if err := c.ShouldBindQuery(&args); err != nil { + ctx.Err = err + return + } + + dockerfileTemplateList, total, err := templateservice.ListDockerfileTemplate(args.PageNum, args.PageSize, ctx.Logger) + resp := ListDockefileResp{ + DockerfileTemplates: dockerfileTemplateList, + Total: total, + } + ctx.Resp = resp + ctx.Err = err +} + +func GetDockerfileTemplateDetail(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = template.GetDockerfileTemplateDetail(c.Param("id"), ctx.Logger) +} + +func DeleteDockerfileTemplate(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + internalhandler.InsertOperationLog(c, ctx.UserName, "", "删除", "模板库-Dockerfile", c.Param("id"), "", ctx.Logger) + + ctx.Err = templateservice.DeleteDockerfileTemplate(c.Param("id"), ctx.Logger) +} + +func GetDockerfileTemplateReference(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = templateservice.GetDockerfileTemplateReference(c.Param("id"), ctx.Logger) +} + +type validateDockerfileTemplateReq struct { + Content string `json:"content"` +} + +type validateDockerfileTemplateResp struct { + Error string `json:"error"` +} + +func ValidateDockerfileTemplate(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + req := &validateDockerfileTemplateReq{} + if err := c.ShouldBindJSON(req); err != nil { + ctx.Err = err + return + } + + err := templateservice.ValidateDockerfileTemplate(req.Content, ctx.Logger) + ctx.Resp = &validateDockerfileTemplateResp{Error: ""} + if err != nil { + ctx.Resp = &validateDockerfileTemplateResp{Error: err.Error()} + } +} diff --git a/pkg/microservice/aslan/core/templatestore/handler/router.go b/pkg/microservice/aslan/core/templatestore/handler/router.go index 120a55ff2d990a689ea13830e999408a8f134cd9..926e1b7dce3629d0f8a5afc6ed1cbc238685bab4 100644 --- a/pkg/microservice/aslan/core/templatestore/handler/router.go +++ b/pkg/microservice/aslan/core/templatestore/handler/router.go @@ -18,26 +18,56 @@ package handler import ( "github.com/gin-gonic/gin" - - ginmiddleware "github.com/koderover/zadig/pkg/middleware/gin" ) type Router struct{} func (*Router) Inject(router *gin.RouterGroup) { - - router.Use(ginmiddleware.Auth()) - // --------------------------------------------------------------------------------------- // chart templates // --------------------------------------------------------------------------------------- - install := router.Group("charts") + chart := router.Group("charts") + { + chart.GET("", ListChartTemplates) + chart.GET("/:name", GetChartTemplate) + chart.GET("/:name/files", ListFiles) + chart.GET("/:name/variables", GetTemplateVariables) + chart.POST("", AddChartTemplate) + chart.PUT("/:name", UpdateChartTemplate) + chart.POST("/:name/reference", SyncChartTemplateReference) + chart.PUT("/:name/variables", UpdateChartTemplateVariables) + chart.DELETE("/:name", RemoveChartTemplate) + } + + dockerfile := router.Group("dockerfile") + { + dockerfile.POST("", CreateDockerfileTemplate) + dockerfile.PUT("/:id", UpdateDockerfileTemplate) + dockerfile.GET("", ListDockerfileTemplate) + dockerfile.GET("/:id", GetDockerfileTemplateDetail) + dockerfile.DELETE("/:id", DeleteDockerfileTemplate) + dockerfile.GET("/:id/reference", GetDockerfileTemplateReference) + dockerfile.POST("/validation", ValidateDockerfileTemplate) + } + + yaml := router.Group("yaml") + { + yaml.POST("", CreateYamlTemplate) + yaml.PUT("/:id", UpdateYamlTemplate) + yaml.GET("", ListYamlTemplate) + yaml.GET("/:id", GetYamlTemplateDetail) + yaml.DELETE("/:id", DeleteYamlTemplate) + yaml.GET("/:id/reference", GetYamlTemplateReference) + yaml.POST("/:id/reference", SyncYamlTemplateReference) + yaml.POST("/getVariables", GetYamlTemplateVariables) + } + + build := router.Group("build") { - install.GET("", ListChartTemplates) - install.GET("/:name", GetChartTemplate) - install.GET("/:name/files", ListFiles) - install.POST("", AddChartTemplate) - install.PUT("/:name", UpdateChartTemplate) - install.DELETE("/:name", RemoveChartTemplate) + build.POST("", AddBuildTemplate) + build.PUT("/:id", UpdateBuildTemplate) + build.GET("", ListBuildTemplates) + build.GET("/:id", GetBuildTemplate) + build.DELETE("/:id", RemoveBuildTemplate) } } diff --git a/pkg/microservice/aslan/core/templatestore/handler/yaml.go b/pkg/microservice/aslan/core/templatestore/handler/yaml.go new file mode 100644 index 0000000000000000000000000000000000000000..673a9a49b7dd5d4f02ada7b5a917bd5d3714d59f --- /dev/null +++ b/pkg/microservice/aslan/core/templatestore/handler/yaml.go @@ -0,0 +1,144 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "encoding/json" + + "github.com/gin-gonic/gin" + + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/template" + templateservice "github.com/koderover/zadig/pkg/microservice/aslan/core/templatestore/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" +) + +func CreateYamlTemplate(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + req := &template.YamlTemplate{} + + if err := c.ShouldBindJSON(req); err != nil { + ctx.Err = err + return + } + + bs, _ := json.Marshal(req) + internalhandler.InsertOperationLog(c, ctx.UserName, "", "创建", "模板-YAML", req.Name, string(bs), ctx.Logger) + + ctx.Err = templateservice.CreateYamlTemplate(req, ctx.Logger) +} + +func UpdateYamlTemplate(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + req := &template.YamlTemplate{} + + if err := c.ShouldBindJSON(req); err != nil { + ctx.Err = err + return + } + + bs, _ := json.Marshal(req) + internalhandler.InsertOperationLog(c, ctx.UserName, "", "更新", "模板-YAML", req.Name, string(bs), ctx.Logger) + + ctx.Err = templateservice.UpdateYamlTemplate(c.Param("id"), req, ctx.Logger) +} + +type listYamlQuery struct { + PageSize int `json:"page_size" form:"page_size,default=100"` + PageNum int `json:"page_num" form:"page_num,default=1"` +} + +type ListYamlResp struct { + SystemVariables []*commonmodels.ChartVariable `json:"system_variables"` + YamlTemplates []*template.YamlListObject `json:"yaml_template"` + Total int `json:"total"` +} + +func ListYamlTemplate(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + // Query Verification + args := &listYamlQuery{} + if err := c.ShouldBindQuery(args); err != nil { + ctx.Err = err + return + } + + systemVariables := templateservice.GetSystemDefaultVariables() + YamlTemplateList, total, err := templateservice.ListYamlTemplate(args.PageNum, args.PageSize, ctx.Logger) + resp := ListYamlResp{ + SystemVariables: systemVariables, + YamlTemplates: YamlTemplateList, + Total: total, + } + ctx.Resp = resp + ctx.Err = err +} + +func GetYamlTemplateDetail(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = templateservice.GetYamlTemplateDetail(c.Param("id"), ctx.Logger) +} + +func DeleteYamlTemplate(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + internalhandler.InsertOperationLog(c, ctx.UserName, "", "删除", "模板-YAML", c.Param("id"), "", ctx.Logger) + + ctx.Err = templateservice.DeleteYamlTemplate(c.Param("id"), ctx.Logger) +} + +func GetYamlTemplateReference(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = templateservice.GetYamlTemplateReference(c.Param("id"), ctx.Logger) +} + +func SyncYamlTemplateReference(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + internalhandler.InsertOperationLog(c, ctx.UserName, "", "同步", "模板-YAML", c.Param("id"), "", ctx.Logger) + + ctx.Err = templateservice.SyncYamlTemplateReference(ctx.UserName, c.Param("id"), ctx.Logger) +} + +type getYamlTemplateVariablesReq struct { + Content string `json:"content"` +} + +func GetYamlTemplateVariables(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + req := &getYamlTemplateVariablesReq{} + if err := c.ShouldBindJSON(req); err != nil { + ctx.Err = err + return + } + + ctx.Resp, ctx.Err = templateservice.GetYamlVariables(req.Content, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/templatestore/service/build.go b/pkg/microservice/aslan/core/templatestore/service/build.go new file mode 100644 index 0000000000000000000000000000000000000000..0ced21cf2274fefeeca6a06b5643f62e0a2a02bc --- /dev/null +++ b/pkg/microservice/aslan/core/templatestore/service/build.go @@ -0,0 +1,115 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "fmt" + + "go.uber.org/zap" + + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + commonutil "github.com/koderover/zadig/pkg/microservice/aslan/core/common/util" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/log" +) + +type BuildTemplateBrief struct { + Id string `json:"id"` + Name string `json:"name"` +} + +type BuildTemplateListResp struct { + BuildTemplates []*BuildTemplateBrief `json:"build_templates"` + Total int `json:"total,omitempty"` +} + +func AddBuildTemplate(userName string, build *commonmodels.BuildTemplate, logger *zap.SugaredLogger) error { + if len(build.Name) == 0 { + return e.ErrCreateBuildModule.AddDesc("empty name") + } + if err := commonutil.CheckDefineResourceParam(build.PreBuild.ResReq, build.PreBuild.ResReqSpec); err != nil { + return e.ErrCreateBuildModule.AddDesc(err.Error()) + } + build.UpdateBy = userName + if err := commonrepo.NewBuildTemplateColl().Create(build); err != nil { + log.Errorf("[Build.Upsert] %s error: %s", build.Name, err) + return e.ErrCreateBuildModule.AddErr(err) + } + return nil +} + +func GetBuildTemplateByName(name string) (*commonmodels.BuildTemplate, error) { + return mongodb.NewBuildTemplateColl().Find(&commonrepo.BuildTemplateQueryOption{Name: name}) +} + +func GetBuildTemplateByID(idStr string) (*commonmodels.BuildTemplate, error) { + return mongodb.NewBuildTemplateColl().Find(&commonrepo.BuildTemplateQueryOption{ID: idStr}) +} + +func ListBuildTemplates(pageNum, pageSize int) (*BuildTemplateListResp, error) { + buildTemplates, count, err := mongodb.NewBuildTemplateColl().List(pageNum, pageSize) + if err != nil { + return nil, err + } + ret := &BuildTemplateListResp{ + Total: count, + } + for _, bt := range buildTemplates { + ret.BuildTemplates = append(ret.BuildTemplates, &BuildTemplateBrief{ + Id: bt.ID.Hex(), + Name: bt.Name, + }) + } + return ret, nil +} + +func RemoveBuildTemplate(id string, logger *zap.SugaredLogger) error { + _, err := GetBuildTemplateByID(id) + if err != nil { + return fmt.Errorf("failed to find build template with id: %s, err: %s", id, err) + } + + // when template build is used by particular builds, this template can't be deleted + usedModules, err := mongodb.NewBuildColl().List(&commonrepo.BuildListOption{ + TemplateID: id, + }) + if err != nil { + return fmt.Errorf("failed to find builds with template id: %s, err: %s", id, err) + } + if len(usedModules) > 0 { + return fmt.Errorf("template build has beed used, can't be deleted") + } + + err = mongodb.NewBuildTemplateColl().DeleteByID(id) + if err != nil { + logger.Errorf("Failed to delete build template %s, err: %s", id, err) + return err + } + return nil +} + +func UpdateBuildTemplate(id string, buildTemplate *commonmodels.BuildTemplate, logger *zap.SugaredLogger) error { + _, err := commonrepo.NewBuildTemplateColl().Find(&commonrepo.BuildTemplateQueryOption{ + ID: id, + }) + if err != nil { + return err + } + return commonrepo.NewBuildTemplateColl().Update(id, buildTemplate) +} diff --git a/pkg/microservice/aslan/core/templatestore/service/chart.go b/pkg/microservice/aslan/core/templatestore/service/chart.go index 5c34db73145eadbe237cd863e740cc28778dc721..2e78e3990b48a99f029d011835770248e366fe3e 100644 --- a/pkg/microservice/aslan/core/templatestore/service/chart.go +++ b/pkg/microservice/aslan/core/templatestore/service/chart.go @@ -19,21 +19,47 @@ package service import ( "fmt" "os" + "path" "path/filepath" + "regexp" + "strings" "github.com/27149chen/afero" + "github.com/pkg/errors" "go.uber.org/zap" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" configbase "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/fs" - "github.com/koderover/zadig/pkg/microservice/aslan/core/templatestore/repository/models" - "github.com/koderover/zadig/pkg/microservice/aslan/core/templatestore/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/template" + "github.com/koderover/zadig/pkg/microservice/aslan/core/service/service" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" + "github.com/koderover/zadig/pkg/tool/log" fsutil "github.com/koderover/zadig/pkg/util/fs" ) -func GetChartTemplate(name string, logger *zap.SugaredLogger) (*Chart, error) { - chart, err := mongodb.NewChartColl().Get(name) +var ( + variableExtractRegexp = regexp.MustCompile("{{.(\\w*)}}") + ChartTemplateDefaultSystemVariable = map[string]string{ + setting.TemplateVariableProduct: setting.TemplateVariableProductDescription, + setting.TemplateVariableService: setting.TemplateVariableServiceDescription, + } +) + +type ChartTemplateListResp struct { + SystemVariables []*commonmodels.ChartVariable `json:"systemVariables"` + ChartTemplates []*template.Chart `json:"chartTemplates"` +} + +func GetChartTemplate(name string, logger *zap.SugaredLogger) (*template.Chart, error) { + chart, err := commonrepo.NewChartColl().Get(name) if err != nil { logger.Errorf("Failed to get chart template %s, err: %s", name, err) return nil, err @@ -41,7 +67,7 @@ func GetChartTemplate(name string, logger *zap.SugaredLogger) (*Chart, error) { localBase := configbase.LocalChartTemplatePath(name) s3Base := configbase.ObjectStorageChartTemplatePath(name) - if err = fs.PreloadFiles(name, localBase, s3Base, logger); err != nil { + if err = fs.PreloadFiles(name, localBase, s3Base, chart.Source, logger); err != nil { return nil, err } @@ -53,53 +79,101 @@ func GetChartTemplate(name string, logger *zap.SugaredLogger) (*Chart, error) { return nil, err } - return &Chart{ + variables := make([]*commonmodels.ChartVariable, 0, len(chart.ChartVariables)) + for _, v := range chart.ChartVariables { + variables = append(variables, &commonmodels.ChartVariable{ + Key: v.Key, + Value: v.Value, + }) + } + + return &template.Chart{ Name: name, CodehostID: chart.CodeHostID, Owner: chart.Owner, Repo: chart.Repo, Path: chart.Path, Branch: chart.Branch, + Namespace: chart.GetNamespace(), Files: fis, + Variables: variables, }, nil } -func ListChartTemplates(logger *zap.SugaredLogger) ([]*Chart, error) { +func GetChartTemplateVariables(name string, logger *zap.SugaredLogger) ([]*commonmodels.ChartVariable, error) { + chart, err := mongodb.NewChartColl().Get(name) + if err != nil { + logger.Errorf("Failed to get chart template %s, err: %s", name, err) + return nil, err + } + + variables := make([]*commonmodels.ChartVariable, 0) + for _, v := range chart.ChartVariables { + variables = append(variables, &commonmodels.ChartVariable{ + Key: v.Key, + Value: v.Value, + }) + } + return variables, nil +} + +func getChartTemplateDefaultVariables() []*commonmodels.ChartVariable { + resp := make([]*commonmodels.ChartVariable, 0) + for key, description := range ChartTemplateDefaultSystemVariable { + resp = append(resp, &commonmodels.ChartVariable{ + Key: key, + Description: description, + }) + } + return resp +} + +func ListChartTemplates(logger *zap.SugaredLogger) (*ChartTemplateListResp, error) { cs, err := mongodb.NewChartColl().List() if err != nil { logger.Errorf("Failed to list chart templates, err: %s", err) return nil, err } - res := make([]*Chart, 0, len(cs)) + res := make([]*template.Chart, 0, len(cs)) for _, c := range cs { - res = append(res, &Chart{ + res = append(res, &template.Chart{ Name: c.Name, CodehostID: c.CodeHostID, Owner: c.Owner, + Namespace: c.GetNamespace(), Repo: c.Repo, Path: c.Path, Branch: c.Branch, }) } - return res, nil + ret := &ChartTemplateListResp{ + SystemVariables: getChartTemplateDefaultVariables(), + ChartTemplates: res, + } + + return ret, nil } -func GetFileContent(name, filePath, fileName string, logger *zap.SugaredLogger) ([]byte, error) { +func GetFileContentForTemplate(name, filePath, fileName string, logger *zap.SugaredLogger) ([]byte, error) { chart, err := mongodb.NewChartColl().Get(name) if err != nil { logger.Errorf("Failed to get chart template %s, err: %s", name, err) return nil, err } + return getFileContent(name, chart.Path, filePath, fileName, chart.Source, logger) +} + +func getFileContent(name, path, filePath, fileName, source string, logger *zap.SugaredLogger) ([]byte, error) { localBase := configbase.LocalChartTemplatePath(name) s3Base := configbase.ObjectStorageChartTemplatePath(name) - if err = fs.PreloadFiles(name, localBase, s3Base, logger); err != nil { + if err := fs.PreloadFiles(name, localBase, s3Base, source, logger); err != nil { return nil, err } - base := filepath.Base(chart.Path) + base := filepath.Base(path) file := filepath.Join(localBase, base, filePath, fileName) fileContent, err := os.ReadFile(file) if err != nil { @@ -110,25 +184,70 @@ func GetFileContent(name, filePath, fileName string, logger *zap.SugaredLogger) return fileContent, nil } +func parseTemplateVariables(name, path, source string, logger *zap.SugaredLogger) ([]string, error) { + valueYamlContent, err := getFileContent(name, path, "", setting.ValuesYaml, source, logger) + if err != nil { + return nil, errors.Wrap(err, "failed to read values.yaml") + } + strSet := sets.NewString() + allMatches := variableExtractRegexp.FindAllStringSubmatch(string(valueYamlContent), -1) + for _, match := range allMatches { + if len(match) != 2 { + continue + } + strSet.Insert(match[1]) + } + return strSet.List(), nil +} + func AddChartTemplate(name string, args *fs.DownloadFromSourceArgs, logger *zap.SugaredLogger) error { if mongodb.NewChartColl().Exist(name) { return fmt.Errorf("a chart template with name %s is already existing", name) } - sha1, err := processChartFromSource(name, args, logger) + ch, err := systemconfig.New().GetCodeHost(args.CodehostID) if err != nil { - logger.Errorf("Failed to create chart %s, err: %s", name, err) + log.Errorf("Failed to get codeHost by id %d, err: %s", args.CodehostID, err) return err } + var ( + sha1 string + loadErr error + ) + switch ch.Type { + case setting.SourceFromGerrit, setting.SourceFromGitee: + sha1, loadErr = processChartFromGitRepo(name, args, logger) + default: + sha1, loadErr = processChartFromSource(name, args, logger) + } + if loadErr != nil { + logger.Errorf("Failed to create chart %s, err: %s", name, loadErr) + return loadErr + } + + variablesNames, err := parseTemplateVariables(name, args.Path, ch.Type, logger) + if err != nil { + return errors.Wrapf(err, "failed to prase variables") + } + + variables := make([]*commonmodels.ChartVariable, 0, len(variablesNames)) + for _, v := range variablesNames { + variables = append(variables, &commonmodels.ChartVariable{ + Key: v, + }) + } return mongodb.NewChartColl().Create(&models.Chart{ - Name: name, - Owner: args.Owner, - Repo: args.Repo, - Path: args.Path, - Branch: args.Branch, - CodeHostID: args.CodehostID, - Sha1: sha1, + Name: name, + Owner: args.Owner, + Namespace: args.Namespace, + Repo: args.Repo, + Path: args.Path, + Branch: args.Branch, + CodeHostID: args.CodehostID, + Sha1: sha1, + ChartVariables: variables, + Source: ch.Type, }) } @@ -138,27 +257,114 @@ func UpdateChartTemplate(name string, args *fs.DownloadFromSourceArgs, logger *z logger.Errorf("Failed to get chart template %s, err: %s", name, err) return err } - - sha1, err := processChartFromSource(name, args, logger) + ch, err := systemconfig.New().GetCodeHost(args.CodehostID) if err != nil { - logger.Errorf("Failed to update chart %s, err: %s", name, err) + log.Errorf("Failed to get codeHost by id %d, err: %s", args.CodehostID, err) return err } + var ( + sha1 string + loadErr error + ) + + switch ch.Type { + case setting.SourceFromGerrit, setting.SourceFromGitee: + sha1, loadErr = processChartFromGitRepo(name, args, logger) + default: + sha1, loadErr = processChartFromSource(name, args, logger) + } + + if loadErr != nil { + logger.Errorf("Failed to update chart %s, err: %s", name, loadErr) + return loadErr + } if chart.Sha1 == sha1 { logger.Debug("Chart %s has no changes, skip updating.", name) return nil } - return mongodb.NewChartColl().Update(&models.Chart{ - Name: name, - Owner: args.Owner, - Repo: args.Repo, - Path: args.Path, - Branch: args.Branch, - CodeHostID: args.CodehostID, - Sha1: sha1, + variablesNames, err := parseTemplateVariables(name, args.Path, ch.Type, logger) + if err != nil { + return errors.Wrapf(err, "failed to prase variables") + } + + variables := make([]*commonmodels.ChartVariable, 0, len(variablesNames)) + curVariableMap := chart.GetVariableMap() + + for _, vName := range variablesNames { + variable := &commonmodels.ChartVariable{ + Key: vName, + } + if v, ok := curVariableMap[vName]; ok { + variable.Value = v.Value + } + variables = append(variables, variable) + } + + err = mongodb.NewChartColl().Update(&models.Chart{ + Name: name, + Owner: args.Owner, + Namespace: args.Namespace, + Repo: args.Repo, + Path: args.Path, + Branch: args.Branch, + CodeHostID: args.CodehostID, + Sha1: sha1, + ChartVariables: variables, + Source: ch.Type, + }) + + return err +} + +func SyncHelmTemplateReference(userName, name string, logger *zap.SugaredLogger) error { + return service.SyncServiceFromTemplate(userName, setting.SourceFromChartTemplate, "", name, logger) +} + +func UpdateChartTemplateVariables(name string, args []*commonmodels.Variable, logger *zap.SugaredLogger) error { + chart, err := mongodb.NewChartColl().Get(name) + if err != nil { + logger.Errorf("Failed to get chart template %s, err: %s", name, err) + return err + } + + requestKeys := sets.NewString() + variables := make([]*commonmodels.ChartVariable, 0) + for _, variable := range args { + variables = append(variables, &commonmodels.ChartVariable{ + Key: variable.Key, + Value: variable.Value, + }) + requestKeys.Insert(variable.Key) + } + + curKeys := sets.NewString() + for _, variable := range chart.ChartVariables { + curKeys.Insert(variable.Key) + } + + if !requestKeys.Equal(curKeys) { + return fmt.Errorf("request variable keys are different from keys in file") + } + + err = mongodb.NewChartColl().Update(&models.Chart{ + Name: name, + Owner: chart.Owner, + Namespace: chart.Namespace, + Repo: chart.Repo, + Path: chart.Path, + Branch: chart.Branch, + CodeHostID: chart.CodeHostID, + Sha1: chart.Sha1, + ChartVariables: variables, }) + + if err != nil { + logger.Errorf("failed to save variables err %s", err) + return fmt.Errorf("failed to save variables") + } + return nil } func RemoveChartTemplate(name string, logger *zap.SugaredLogger) error { @@ -168,7 +374,7 @@ func RemoveChartTemplate(name string, logger *zap.SugaredLogger) error { return err } - if err = fs.DeleteArchivedFileFromS3(name, configbase.ObjectStorageChartTemplatePath(name), logger); err != nil { + if err = fs.DeleteArchivedFileFromS3([]string{name}, configbase.ObjectStorageChartTemplatePath(name), logger); err != nil { logger.Warnf("Failed to delete file %s, err: %s", name, err) } @@ -196,7 +402,82 @@ func processChartFromSource(name string, args *fs.DownloadFromSourceArgs, logger localBase := configbase.LocalChartTemplatePath(name) s3Base := configbase.ObjectStorageChartTemplatePath(name) - err1 := fs.SaveAndUploadFiles(tree, name, localBase, s3Base, logger) + err = os.RemoveAll(localBase) + if err != nil { + log.Errorf("failed to remove current template dir: %s, err: %s", localBase, err) + return + } + + err1 := fs.SaveAndUploadFiles(tree, []string{name}, localBase, s3Base, logger) + if err1 != nil { + logger.Errorf("Failed to save files to disk, err: %s", err1) + err = err1 + return + } + + logger.Debug("Finish to save and upload chart") + }) + + wg.Start(func() { + logger.Debug("Start to calculate sha1 for chart") + tmpDir, err1 := os.MkdirTemp("", "") + if err1 != nil { + logger.Errorf("Failed to create temp dir, err: %s", err1) + err = err1 + return + } + defer func() { + _ = os.RemoveAll(tmpDir) + }() + + fileName := fmt.Sprintf("%s.tar.gz", filepath.Base(args.Path)) + tarball := filepath.Join(tmpDir, fileName) + if err1 = fsutil.Tar(tree, tarball); err1 != nil { + logger.Errorf("Failed to archive files to %s, err: %s", tarball, err1) + err = err1 + return + } + + sha1, err1 = fsutil.Sha1(os.DirFS(tmpDir), fileName) + if err1 != nil { + logger.Errorf("Failed to calculate sha1 for file %s, err: %s", tarball, err1) + err = err1 + return + } + + logger.Debug("Finish to calculate sha1 for chart") + }) + + wg.Wait() + + if err != nil { + return "", err + } + + return sha1, nil +} + +func processChartFromGitRepo(name string, args *fs.DownloadFromSourceArgs, logger *zap.SugaredLogger) (string, error) { + var ( + wg wait.Group + sha1 string + err error + ) + + base := path.Join(config.S3StoragePath(), args.Repo) + filePath := strings.TrimLeft(args.Path, "/") + currentChartPath := path.Join(base, filePath) + wg.Start(func() { + logger.Debug("Start to save and upload chart") + localBase := configbase.LocalChartTemplatePath(name) + + err = os.RemoveAll(path.Join(localBase, path.Base(args.Path))) + if err != nil { + log.Errorf("failed to remove current template dir: %s, err: %s", localBase, err) + return + } + + err1 := fs.CopyAndUploadFiles([]string{}, path.Join(localBase, path.Base(args.Path)), "", currentChartPath, logger) if err1 != nil { logger.Errorf("Failed to save files to disk, err: %s", err1) err = err1 @@ -220,6 +501,7 @@ func processChartFromSource(name string, args *fs.DownloadFromSourceArgs, logger fileName := fmt.Sprintf("%s.tar.gz", filepath.Base(args.Path)) tarball := filepath.Join(tmpDir, fileName) + tree := os.DirFS(currentChartPath) if err1 = fsutil.Tar(tree, tarball); err1 != nil { logger.Errorf("Failed to archive files to %s, err: %s", tarball, err1) err = err1 diff --git a/pkg/microservice/aslan/core/templatestore/service/dockerfile.go b/pkg/microservice/aslan/core/templatestore/service/dockerfile.go new file mode 100644 index 0000000000000000000000000000000000000000..6b0282e0fe19d6bc5a8eaf9f2477e599e2708d13 --- /dev/null +++ b/pkg/microservice/aslan/core/templatestore/service/dockerfile.go @@ -0,0 +1,117 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "errors" + "strings" + + dockerfileinstructions "github.com/moby/buildkit/frontend/dockerfile/instructions" + "github.com/moby/buildkit/frontend/dockerfile/parser" + "go.uber.org/zap" + + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/template" +) + +func CreateDockerfileTemplate(template *template.DockerfileTemplate, logger *zap.SugaredLogger) error { + err := commonrepo.NewDockerfileTemplateColl().Create(&commonmodels.DockerfileTemplate{ + Name: template.Name, + Content: template.Content, + }) + if err != nil { + logger.Errorf("create dockerfile template error: %s", err) + } + return err +} + +func UpdateDockerfileTemplate(id string, template *template.DockerfileTemplate, logger *zap.SugaredLogger) error { + err := commonrepo.NewDockerfileTemplateColl().Update( + id, + &commonmodels.DockerfileTemplate{ + Name: template.Name, + Content: template.Content, + }, + ) + if err != nil { + logger.Errorf("update dockerfile template error: %s", err) + } + return err +} + +func ListDockerfileTemplate(pageNum, pageSize int, logger *zap.SugaredLogger) ([]*template.DockerfileListObject, int, error) { + resp := make([]*template.DockerfileListObject, 0) + templateList, total, err := commonrepo.NewDockerfileTemplateColl().List(pageNum, pageSize) + if err != nil { + logger.Errorf("list dockerfile template error: %s", err) + return resp, 0, err + } + for _, obj := range templateList { + resp = append(resp, &template.DockerfileListObject{ + ID: obj.ID.Hex(), + Name: obj.Name, + }) + } + return resp, total, err +} + +func DeleteDockerfileTemplate(id string, logger *zap.SugaredLogger) error { + ref, err := commonrepo.NewBuildColl().GetDockerfileTemplateReference(id) + if err != nil { + logger.Errorf("Failed to get build reference for template id: %s, the error is: %s", id, err) + return err + } + if len(ref) > 0 { + return errors.New("this template is in use") + } + err = commonrepo.NewDockerfileTemplateColl().DeleteByID(id) + if err != nil { + logger.Errorf("Failed to delete dockerfile template of id: %s, the error is: %s", id, err) + } + return err +} + +func GetDockerfileTemplateReference(id string, logger *zap.SugaredLogger) ([]*template.BuildReference, error) { + ret := make([]*template.BuildReference, 0) + referenceList, err := commonrepo.NewBuildColl().GetDockerfileTemplateReference(id) + if err != nil { + logger.Errorf("Failed to get build reference for dockerfile template id: %s, the error is: %s", id, err) + return ret, err + } + for _, reference := range referenceList { + ret = append(ret, &template.BuildReference{ + BuildName: reference.Name, + ProjectName: reference.ProductName, + }) + } + return ret, nil +} + +func ValidateDockerfileTemplate(template string, _ *zap.SugaredLogger) error { + // some dockerfile validation stuff + reader := strings.NewReader(template) + result, err := parser.Parse(reader) + if err != nil { + return err + } + _, _, err = dockerfileinstructions.Parse(result.AST) + if err != nil { + return err + } + return nil +} diff --git a/pkg/microservice/aslan/core/templatestore/service/yaml.go b/pkg/microservice/aslan/core/templatestore/service/yaml.go new file mode 100644 index 0000000000000000000000000000000000000000..0716647d293270a749c8e40dd7e9811c20e264dd --- /dev/null +++ b/pkg/microservice/aslan/core/templatestore/service/yaml.go @@ -0,0 +1,188 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "errors" + "regexp" + "strings" + + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/template" + "github.com/koderover/zadig/pkg/microservice/aslan/core/service/service" + "github.com/koderover/zadig/pkg/setting" +) + +var DefaultSystemVariable = map[string]string{ + setting.TemplateVariableProduct: setting.TemplateVariableProductDescription, + setting.TemplateVariableService: setting.TemplateVariableServiceDescription, +} + +func CreateYamlTemplate(template *template.YamlTemplate, logger *zap.SugaredLogger) error { + vars := make([]*models.Variable, 0) + for _, variable := range template.Variable { + vars = append(vars, &models.Variable{ + Key: variable.Key, + Value: variable.Value, + }) + } + err := commonrepo.NewYamlTemplateColl().Create(&models.YamlTemplate{ + Name: template.Name, + Content: template.Content, + Variables: vars, + }) + if err != nil { + logger.Errorf("create dockerfile template error: %s", err) + } + return err +} + +func UpdateYamlTemplate(id string, template *template.YamlTemplate, logger *zap.SugaredLogger) error { + vars := make([]*models.Variable, 0) + for _, variable := range template.Variable { + vars = append(vars, &models.Variable{ + Key: variable.Key, + Value: variable.Value, + }) + } + err := commonrepo.NewYamlTemplateColl().Update( + id, + &models.YamlTemplate{ + Name: template.Name, + Content: template.Content, + Variables: vars, + }, + ) + if err != nil { + logger.Errorf("update dockerfile template error: %s", err) + } + return err +} + +func ListYamlTemplate(pageNum, pageSize int, logger *zap.SugaredLogger) ([]*template.YamlListObject, int, error) { + resp := make([]*template.YamlListObject, 0) + templateList, total, err := commonrepo.NewYamlTemplateColl().List(pageNum, pageSize) + if err != nil { + logger.Errorf("list dockerfile template error: %s", err) + return resp, 0, err + } + for _, obj := range templateList { + resp = append(resp, &template.YamlListObject{ + ID: obj.ID.Hex(), + Name: obj.Name, + }) + } + return resp, total, err +} + +func GetYamlTemplateDetail(id string, logger *zap.SugaredLogger) (*template.YamlDetail, error) { + resp := new(template.YamlDetail) + yamlTemplate, err := commonrepo.NewYamlTemplateColl().GetById(id) + if err != nil { + logger.Errorf("Failed to get dockerfile template from id: %s, the error is: %s", id, err) + return nil, err + } + variables := make([]*models.ChartVariable, 0) + for _, v := range yamlTemplate.Variables { + variables = append(variables, &models.ChartVariable{ + Key: v.Key, + Value: v.Value, + }) + } + resp.ID = yamlTemplate.ID.Hex() + resp.Name = yamlTemplate.Name + resp.Content = yamlTemplate.Content + resp.Variables = variables + return resp, nil +} + +func DeleteYamlTemplate(id string, logger *zap.SugaredLogger) error { + ref, err := commonrepo.NewServiceColl().GetTemplateReference(id) + if err != nil { + logger.Errorf("Failed to get service reference for template id: %s, the error is: %s", id, err) + return err + } + if len(ref) > 0 { + return errors.New("this template is in use") + } + err = commonrepo.NewYamlTemplateColl().DeleteByID(id) + if err != nil { + logger.Errorf("Failed to delete dockerfile template of id: %s, the error is: %s", id, err) + } + return err +} + +func SyncYamlTemplateReference(userName, id string, logger *zap.SugaredLogger) error { + return service.SyncServiceFromTemplate(userName, setting.ServiceSourceTemplate, id, "", logger) +} + +func GetYamlTemplateReference(id string, logger *zap.SugaredLogger) ([]*template.ServiceReference, error) { + ret := make([]*template.ServiceReference, 0) + referenceList, err := commonrepo.NewServiceColl().GetTemplateReference(id) + if err != nil { + logger.Errorf("Failed to get build reference for dockerfile template id: %s, the error is: %s", id, err) + return ret, err + } + for _, reference := range referenceList { + ret = append(ret, &template.ServiceReference{ + ServiceName: reference.ServiceName, + ProjectName: reference.ProductName, + }) + } + return ret, nil +} + +func GetYamlVariables(s string, logger *zap.SugaredLogger) ([]*models.ChartVariable, error) { + resp := make([]*models.ChartVariable, 0) + regex, err := regexp.Compile(setting.RegExpParameter) + if err != nil { + logger.Errorf("Cannot get regexp from the expression: %s, the error is: %s", setting.RegExpParameter, err) + return []*models.ChartVariable{}, err + } + params := regex.FindAllString(s, -1) + keyMap := make(map[string]int) + for _, param := range params { + key := getParameterKey(param) + if keyMap[key] == 0 { + resp = append(resp, &models.ChartVariable{ + Key: key, + }) + keyMap[key] = 1 + } + } + return resp, nil +} + +func GetSystemDefaultVariables() []*models.ChartVariable { + resp := make([]*models.ChartVariable, 0) + for key, description := range DefaultSystemVariable { + resp = append(resp, &models.ChartVariable{ + Key: key, + Description: description, + }) + } + return resp +} + +// getParameter +func getParameterKey(parameter string) string { + a := strings.TrimPrefix(parameter, "{{.") + return strings.TrimSuffix(a, "}}") +} diff --git a/pkg/microservice/aslan/core/workflow/handler/build.go b/pkg/microservice/aslan/core/workflow/handler/build.go index c3c2b7f6a5e1f441295c684902601e4a13752539..37166728d3e2de4f1f1cf920cb58f2063aef6969 100644 --- a/pkg/microservice/aslan/core/workflow/handler/build.go +++ b/pkg/microservice/aslan/core/workflow/handler/build.go @@ -19,6 +19,7 @@ package handler import ( "github.com/gin-gonic/gin" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" internalhandler "github.com/koderover/zadig/pkg/shared/handler" ) @@ -27,7 +28,11 @@ func BuildModuleToSubTasks(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - resp, err := workflow.BuildModuleToSubTasks(c.Param("name"), "", "", c.Query("productName"), nil, nil, ctx.Logger) + args := &models.BuildModuleArgs{ + BuildName: c.Param("name"), + ProductName: c.Query("projectName"), + } + resp, err := workflow.BuildModuleToSubTasks(args, ctx.Logger) if err != nil { ctx.Err = err return diff --git a/pkg/microservice/aslan/core/workflow/handler/bundle.go b/pkg/microservice/aslan/core/workflow/handler/bundle.go new file mode 100644 index 0000000000000000000000000000000000000000..95117076156b313d63afd7e8219c445352f9ee3c --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/handler/bundle.go @@ -0,0 +1,31 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" +) + +func GetBundleResources(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = workflow.GetBundleResources(ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/workflow/handler/favorate_pipeline.go b/pkg/microservice/aslan/core/workflow/handler/favorate_pipeline.go index bbc4dc59be71f7460e092a20f671cbd5da525d76..0574ffd50e3cece79865d1594da30b71f1146bfa 100644 --- a/pkg/microservice/aslan/core/workflow/handler/favorate_pipeline.go +++ b/pkg/microservice/aslan/core/workflow/handler/favorate_pipeline.go @@ -35,13 +35,13 @@ func ListFavoritePipelines(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - productName := c.Query("productName") + productName := c.Query("projectName") workflowType := c.Query("type") if workflowType == "" { ctx.Err = e.ErrInvalidParam.AddDesc("type can't be empty!") return } - ctx.Resp, ctx.Err = workflow.ListFavoritePipelines(&commonrepo.FavoriteArgs{UserID: ctx.User.ID, ProductName: productName, Type: workflowType}) + ctx.Resp, ctx.Err = workflow.ListFavoritePipelines(&commonrepo.FavoriteArgs{UserID: ctx.UserID, ProductName: productName, Type: workflowType}) } func DeleteFavoritePipeline(c *gin.Context) { @@ -55,8 +55,8 @@ func DeleteFavoritePipeline(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("productName or name or type can't be empty!") return } - internalhandler.InsertOperationLog(c, ctx.Username, productName, "删除", "收藏工作流", workflowName, "", ctx.Logger) - ctx.Err = workflow.DeleteFavoritePipeline(&commonrepo.FavoriteArgs{UserID: ctx.User.ID, ProductName: productName, Type: workflowType, Name: workflowName}) + internalhandler.InsertOperationLog(c, ctx.UserName, productName, "删除", "收藏工作流", workflowName, "", ctx.Logger) + ctx.Err = workflow.DeleteFavoritePipeline(&commonrepo.FavoriteArgs{UserID: ctx.UserID, ProductName: productName, Type: workflowType, Name: workflowName}) } func CreateFavoritePipeline(c *gin.Context) { @@ -71,14 +71,14 @@ func CreateFavoritePipeline(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("CreateFavoritePipeline json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, args.ProductName, "新增", "收藏工作流", args.Name, string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, args.ProductName, "新增", "收藏工作流", args.Name, string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) if err := c.BindJSON(args); err != nil { ctx.Err = e.ErrInvalidParam.AddDesc("invalid Favorite json args") return } - args.UserID = ctx.User.ID + args.UserID = ctx.UserID ctx.Err = workflow.CreateFavoritePipeline(args, ctx.Logger) } diff --git a/pkg/microservice/aslan/core/workflow/handler/pipeline.go b/pkg/microservice/aslan/core/workflow/handler/pipeline.go index cf79ec3e937187b89f9a5666d257d2680c48f4eb..ca3279631b34849a5a28e6e3fced8d55c50e1a04 100644 --- a/pkg/microservice/aslan/core/workflow/handler/pipeline.go +++ b/pkg/microservice/aslan/core/workflow/handler/pipeline.go @@ -54,7 +54,7 @@ func GetProductNameByPipeline(c *gin.Context) { if pipelineName == "" { pipelineName = c.Param("name") } - pipeline, err := workflow.GetPipeline(ctx.User.ID, pipelineName, ctx.Logger) + pipeline, err := workflow.GetPipeline(ctx.UserID, pipelineName, ctx.Logger) if err != nil { log.Errorf("GetProductNameByPipeline err : %v", err) return @@ -85,7 +85,7 @@ func GetPipeline(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = workflow.GetPipeline(ctx.User.ID, c.Param("name"), ctx.Logger) + ctx.Resp, ctx.Err = workflow.GetPipeline(ctx.UserID, c.Param("name"), ctx.Logger) } // UpsertPipeline create a new pipeline @@ -101,7 +101,7 @@ func UpsertPipeline(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("UpsertPipeline json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, args.ProductName, "新增", "单服务-工作流", args.Name, string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, args.ProductName, "新增", "单服务-工作流", args.Name, string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) if err := c.BindJSON(args); err != nil || len(args.Name) == 0 { @@ -109,7 +109,7 @@ func UpsertPipeline(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc(fmt.Sprintf("invalid pipeline json args: %v", err)) return } - args.UpdateBy = ctx.Username + args.UpdateBy = ctx.UserName ctx.Err = workflow.UpsertPipeline(args, ctx.Logger) } @@ -118,8 +118,8 @@ func CopyPipeline(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - internalhandler.InsertOperationLog(c, ctx.Username, c.GetString("productName"), "复制", "单服务-工作流", fmt.Sprintf("old:%s,new:%s", c.Param("old"), c.Param("new")), "", ctx.Logger) - ctx.Err = workflow.CopyPipeline(c.Param("old"), c.Param("new"), ctx.Username, ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, c.GetString("productName"), "复制", "单服务-工作流", fmt.Sprintf("old:%s,new:%s", c.Param("old"), c.Param("new")), "", ctx.Logger) + ctx.Err = workflow.CopyPipeline(c.Param("old"), c.Param("new"), ctx.UserName, ctx.Logger) } // RenamePipeline rename pipeline @@ -127,7 +127,7 @@ func RenamePipeline(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - internalhandler.InsertOperationLog(c, ctx.Username, c.GetString("productName"), "修改", "单服务-工作流", fmt.Sprintf("old:%s,new:%s", c.Param("old"), c.Param("new")), "", ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, c.GetString("productName"), "修改", "单服务-工作流", fmt.Sprintf("old:%s,new:%s", c.Param("old"), c.Param("new")), "", ctx.Logger) ctx.Err = workflow.RenamePipeline(c.Param("old"), c.Param("new"), ctx.Logger) } @@ -136,6 +136,6 @@ func DeletePipeline(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - internalhandler.InsertOperationLog(c, ctx.Username, c.GetString("productName"), "删除", "单服务-工作流", c.Param("name"), "", ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, c.GetString("productName"), "删除", "单服务-工作流", c.Param("name"), "", ctx.Logger) ctx.Err = commonservice.DeletePipeline(c.Param("name"), ctx.RequestID, false, ctx.Logger) } diff --git a/pkg/microservice/aslan/core/workflow/handler/pipeline_status.go b/pkg/microservice/aslan/core/workflow/handler/pipeline_status.go index be25decbe2ccc3f932ff900cf81d6d6d6cb2d434..a171d0e0d1ed6862d60d3dc8a83f4ad1a96b246f 100644 --- a/pkg/microservice/aslan/core/workflow/handler/pipeline_status.go +++ b/pkg/microservice/aslan/core/workflow/handler/pipeline_status.go @@ -28,7 +28,7 @@ func ListPipelinesPreview(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = workflow.ListPipelinesPreview(ctx.User.ID, ctx.Logger) + ctx.Resp, ctx.Err = workflow.ListPipelinesPreview(ctx.UserID, ctx.Logger) } // find task by commitId diff --git a/pkg/microservice/aslan/core/workflow/handler/pipeline_task.go b/pkg/microservice/aslan/core/workflow/handler/pipeline_task.go index 40b5f9dc69f1ab3a9168c5c308bc7883a08f94ca..38e94b0953d49085ff26add36745807edb9b4293 100644 --- a/pkg/microservice/aslan/core/workflow/handler/pipeline_task.go +++ b/pkg/microservice/aslan/core/workflow/handler/pipeline_task.go @@ -29,6 +29,7 @@ import ( commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" + "github.com/koderover/zadig/pkg/setting" internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" @@ -47,7 +48,7 @@ func GetProductNameByPipelineTask(c *gin.Context) { return } - pipeline, err := workflow.GetPipeline(ctx.User.ID, args.PipelineName, ctx.Logger) + pipeline, err := workflow.GetPipeline(ctx.UserID, args.PipelineName, ctx.Logger) if err != nil { log.Errorf("GetProductNameByPipelineTask err : %v", err) return @@ -70,7 +71,7 @@ func CreatePipelineTask(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("CreatePipelineTask json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, c.GetString("productName"), "新增", "单服务-工作流task", args.PipelineName, string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, c.GetString("productName"), "新增", "单服务-工作流task", args.PipelineName, string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) if err := c.BindJSON(args); err != nil { @@ -78,7 +79,7 @@ func CreatePipelineTask(c *gin.Context) { return } if args.TaskCreator == "" { - args.TaskCreator = ctx.Username + args.TaskCreator = ctx.UserName } args.ReqID = ctx.RequestID @@ -86,7 +87,7 @@ func CreatePipelineTask(c *gin.Context) { // 发送通知 if ctx.Err != nil { - commonservice.SendFailedTaskMessage(ctx.Username, args.ProductName, args.PipelineName, ctx.RequestID, config.SingleType, ctx.Err, ctx.Logger) + commonservice.SendFailedTaskMessage(ctx.UserName, args.ProductName, args.PipelineName, ctx.RequestID, config.SingleType, ctx.Err, ctx.Logger) } } @@ -106,7 +107,7 @@ func ListPipelineTasksResult(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid start at number") return } - ctx.Resp, ctx.Err = workflow.ListPipelineTasksV2Result(c.Param("name"), config.SingleType, maxResult, startAt, ctx.Logger) + ctx.Resp, ctx.Err = workflow.ListPipelineTasksV2Result(c.Param("name"), config.SingleType, "", []string{}, maxResult, startAt, ctx.Logger) } func GetPipelineTask(c *gin.Context) { @@ -125,7 +126,7 @@ func GetPipelineTask(c *gin.Context) { func RestartPipelineTask(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - internalhandler.InsertOperationLog(c, ctx.Username, c.GetString("productName"), "重启", "单服务-工作流task", c.Param("name"), "", ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, c.Query("projectName"), "重启", "单服务-工作流task", c.Param("name"), "", ctx.Logger) taskID, err := strconv.ParseInt(c.Param("id"), 10, 64) if err != nil { @@ -133,20 +134,20 @@ func RestartPipelineTask(c *gin.Context) { return } - ctx.Err = workflow.RestartPipelineTaskV2(ctx.Username, taskID, c.Param("name"), config.SingleType, ctx.Logger) + ctx.Err = workflow.RestartPipelineTaskV2(ctx.UserName, taskID, c.Param("name"), config.SingleType, ctx.Logger) } func CancelTaskV2(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - internalhandler.InsertOperationLog(c, ctx.Username, c.GetString("productName"), "取消", "单服务-工作流task", c.Param("name"), "", ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, c.Query("projectName"), "取消", "单服务-工作流task", c.Param("name"), "", ctx.Logger) taskID, err := strconv.ParseInt(c.Param("id"), 10, 64) if err != nil { ctx.Err = e.ErrInvalidParam.AddDesc("invalid task id") return } - ctx.Err = commonservice.CancelTaskV2(ctx.Username, c.Param("name"), taskID, config.SingleType, ctx.RequestID, ctx.Logger) + ctx.Err = commonservice.CancelTaskV2(ctx.UserName, c.Param("name"), taskID, config.SingleType, ctx.RequestID, ctx.Logger) } // ListPipelineUpdatableProductNames 启动任务时检查部署环境 @@ -154,7 +155,7 @@ func ListPipelineUpdatableProductNames(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = workflow.ListPipelineUpdatableProductNames(ctx.Username, c.Param("name"), ctx.Logger) + ctx.Resp, ctx.Err = workflow.ListPipelineUpdatableProductNames(ctx.UserName, c.Param("name"), ctx.Logger) } func GetPackageFile(c *gin.Context) { @@ -191,12 +192,22 @@ func GetArtifactFile(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid task id") return } + notHistoryFileFlag, err := strconv.ParseBool(c.DefaultQuery("notHistoryFileFlag", "false")) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid notHistoryFileFlag") + return + } - resp, err := workflow.GetArtifactFileContent(c.Param("pipelineName"), taskID, ctx.Logger) + resp, err := workflow.GetArtifactFileContent(c.Param("pipelineName"), taskID, notHistoryFileFlag, ctx.Logger) if err != nil { ctx.Err = err return } - c.Writer.Header().Set("Content-Disposition", `attachment; filename="artifact.tar.gz"`) + if notHistoryFileFlag { + c.Writer.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, setting.ArtifactResultOut)) + } else { + c.Writer.Header().Set("Content-Disposition", `attachment; filename="artifact.tar.gz"`) + } + c.Data(200, "application/octet-stream", resp) } diff --git a/pkg/microservice/aslan/core/workflow/handler/plugin_repo.go b/pkg/microservice/aslan/core/workflow/handler/plugin_repo.go new file mode 100644 index 0000000000000000000000000000000000000000..9c66c515f296e7854244734387e922ff73b5b5be --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/handler/plugin_repo.go @@ -0,0 +1,31 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" +) + +func ListPluginTemplates(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = workflow.ListPluginTemplates(ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/workflow/handler/router.go b/pkg/microservice/aslan/core/workflow/handler/router.go index fa511c593c8a5ad66704a11006488f12b400e257..99f8230b4ae3508b5a1d13c75fe58bab6a9396ca 100644 --- a/pkg/microservice/aslan/core/workflow/handler/router.go +++ b/pkg/microservice/aslan/core/workflow/handler/router.go @@ -18,9 +18,6 @@ package handler import ( "github.com/gin-gonic/gin" - - gin2 "github.com/koderover/zadig/pkg/middleware/gin" - "github.com/koderover/zadig/pkg/types/permission" ) type Router struct{} @@ -34,8 +31,6 @@ func (*Router) Inject(router *gin.RouterGroup) { webhook.POST("", ProcessWebHook) } - router.Use(gin2.Auth()) - build := router.Group("build") { build.GET("/:name/:version/to/subtasks", BuildModuleToSubTasks) @@ -49,7 +44,10 @@ func (*Router) Inject(router *gin.RouterGroup) { sse.GET("/workflows/id/:id/pipelines/:name", GetWorkflowTaskSSE) sse.GET("/tasks/running", RunningPipelineTasksSSE) sse.GET("/tasks/pending", PendingPipelineTasksSSE) + sse.GET("/workflowTasks/running", RunningWorkflowTasksSSE) + sse.GET("/workflowTasks/pending", PendingWorkflowTasksSSE) sse.GET("/tasks/id/:id/pipelines/:name", GetPipelineTaskSSE) + sse.GET("/workflowtask/v3/id/:id/name/:name", GetWorkflowTaskV3SSE) } // --------------------------------------------------------------------------------------- @@ -59,10 +57,10 @@ func (*Router) Inject(router *gin.RouterGroup) { { pipeline.GET("", ListPipelines) pipeline.GET("/:name", GetPipeline) - pipeline.POST("", GetPipelineProductName, gin2.IsHavePermission([]string{permission.WorkflowCreateUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, UpsertPipeline) - pipeline.POST("/old/:old/new/:new", GetProductNameByPipeline, gin2.IsHavePermission([]string{permission.WorkflowCreateUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, CopyPipeline) - pipeline.PUT("/rename/:old/:new", GetProductNameByPipeline, gin2.IsHavePermission([]string{permission.WorkflowUpdateUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, RenamePipeline) - pipeline.DELETE("/:name", GetProductNameByPipeline, gin2.IsHavePermission([]string{permission.WorkflowDeleteUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, DeletePipeline) + pipeline.POST("", GetPipelineProductName, UpsertPipeline) + pipeline.POST("/old/:old/new/:new", GetProductNameByPipeline, CopyPipeline) + pipeline.PUT("/rename/:old/:new", GetProductNameByPipeline, RenamePipeline) + pipeline.DELETE("/:name", GetProductNameByPipeline, DeletePipeline) } // --------------------------------------------------------------------------------------- @@ -79,11 +77,11 @@ func (*Router) Inject(router *gin.RouterGroup) { // --------------------------------------------------------------------------------------- taskV2 := router.Group("v2/tasks") { - taskV2.POST("", GetProductNameByPipelineTask, gin2.IsHavePermission([]string{permission.WorkflowTaskUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, CreatePipelineTask) + taskV2.POST("", GetProductNameByPipelineTask, CreatePipelineTask) taskV2.GET("/max/:max/start/:start/pipelines/:name", ListPipelineTasksResult) taskV2.GET("/id/:id/pipelines/:name", GetPipelineTask) - taskV2.POST("/id/:id/pipelines/:name/restart", GetProductNameByPipeline, gin2.IsHavePermission([]string{permission.WorkflowTaskUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, RestartPipelineTask) - taskV2.DELETE("/id/:id/pipelines/:name", GetProductNameByPipeline, gin2.IsHavePermission([]string{permission.WorkflowTaskUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, CancelTaskV2) + taskV2.POST("/id/:id/pipelines/:name/restart", GetProductNameByPipeline, RestartPipelineTask) + taskV2.DELETE("/id/:id/pipelines/:name", GetProductNameByPipeline, CancelTaskV2) taskV2.GET("/pipelines/:name/products", ListPipelineUpdatableProductNames) taskV2.GET("/file", GetPackageFile) taskV2.GET("/workflow/:pipelineName/taskId/:taskId", GetArtifactFile) @@ -94,8 +92,8 @@ func (*Router) Inject(router *gin.RouterGroup) { // --------------------------------------------------------------------------------------- favorite := router.Group("favorite") { - favorite.POST("", gin2.UpdateOperationLogStatus, CreateFavoritePipeline) - favorite.DELETE("/:productName/:name/:type", gin2.UpdateOperationLogStatus, DeleteFavoritePipeline) + favorite.POST("", CreateFavoritePipeline) + favorite.DELETE("/:productName/:name/:type", DeleteFavoritePipeline) favorite.GET("", ListFavoritePipelines) } @@ -105,12 +103,12 @@ func (*Router) Inject(router *gin.RouterGroup) { workflow := router.Group("workflow") { workflow.POST("/:productName/auto", AutoCreateWorkflow) - workflow.POST("", GetWorkflowProductName, gin2.IsHavePermission([]string{permission.WorkflowCreateUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, CreateWorkflow) - workflow.PUT("", GetWorkflowProductName, gin2.IsHavePermission([]string{permission.WorkflowUpdateUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, UpdateWorkflow) + workflow.POST("", GetWorkflowProductName, CreateWorkflow) + workflow.PUT("/:workflowName", GetWorkflowProductName, UpdateWorkflow) workflow.GET("", ListWorkflows) - workflow.GET("/testName/:testName", ListAllWorkflows) + workflow.GET("/testName/:testName", ListTestWorkflows) workflow.GET("/find/:name", FindWorkflow) - workflow.DELETE("/:name", GetProductNameByWorkflow, gin2.IsHavePermission([]string{permission.WorkflowDeleteUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, DeleteWorkflow) + workflow.DELETE("/:name", GetProductNameByWorkflow, DeleteWorkflow) workflow.GET("/preset/:productName", PreSetWorkflow) workflow.PUT("/old/:old/new/:new", CopyWorkflow) @@ -124,16 +122,84 @@ func (*Router) Inject(router *gin.RouterGroup) { //todo 修改权限的uuid workflowtask.GET("/targets/:productName/:namespace", GetWorkflowArgs) workflowtask.GET("/preset/:namespace/:workflowName", PresetWorkflowArgs) - workflowtask.POST("", GetWorkflowTaskProductName, gin2.IsHavePermission([]string{permission.WorkflowTaskUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, CreateWorkflowTask) - workflowtask.PUT("", GetWorkflowTaskProductName, gin2.IsHavePermission([]string{permission.WorkflowTaskUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, CreateArtifactWorkflowTask) + workflowtask.POST("/:id", CreateWorkflowTask) + workflowtask.PUT("/:id", CreateArtifactWorkflowTask) workflowtask.GET("/max/:max/start/:start/pipelines/:name", ListWorkflowTasksResult) + workflowtask.GET("/filters/pipelines/:name", GetFiltersPipeline) workflowtask.GET("/id/:id/pipelines/:name", GetWorkflowTask) - workflowtask.POST("/id/:id/pipelines/:name/restart", GetWorkflowTaskProductNameByTask, gin2.IsHavePermission([]string{permission.WorkflowTaskUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, RestartWorkflowTask) - workflowtask.DELETE("/id/:id/pipelines/:name", GetWorkflowTaskProductNameByTask, gin2.IsHavePermission([]string{permission.WorkflowTaskUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, CancelWorkflowTaskV2) + workflowtask.POST("/id/:id/pipelines/:name/restart", RestartWorkflowTask) + workflowtask.DELETE("/id/:id/pipelines/:name", CancelWorkflowTaskV2) + workflowtask.GET("/callback/id/:id/name/:name", GetWorkflowTaskCallback) } serviceTask := router.Group("servicetask") { serviceTask.GET("/workflows/:productName/:envName/:serviceName/:serviceType", ListServiceWorkflows) } + + // --------------------------------------------------------------------------------------- + // 新版本 通用工作流（暂命名） 接口 + // --------------------------------------------------------------------------------------- + workflowV3 := router.Group("v3") + { + workflowV3.POST("", CreateWorkflowV3) + workflowV3.GET("", ListWorkflowsV3) + workflowV3.GET("/:id", GetWorkflowV3Detail) + workflowV3.PUT("/:id", UpdateWorkflowV3) + workflowV3.DELETE("/:id", DeleteWorkflowV3) + workflowV3.GET("/:id/args", GetWorkflowV3Args) + } + + // --------------------------------------------------------------------------------------- + // workflow v3 任务接口 + // --------------------------------------------------------------------------------------- + taskV3 := router.Group("v3/workflowtask") + { + taskV3.POST("", CreateWorkflowTaskV3) + taskV3.POST("/id/:id/name/:name/restart", RestartWorkflowTaskV3) + taskV3.DELETE("/id/:id/name/:name", CancelWorkflowTaskV3) + taskV3.GET("/max/:max/start/:start/name/:name", ListWorkflowV3TasksResult) + taskV3.GET("/id/:id/name/:name", GetWorkflowTaskV3) + taskV3.GET("/callback/id/:id/name/:name", GetWorkflowTaskV3Callback) + } + + // --------------------------------------------------------------------------------------- + // 新版本 通用工作流（暂命名） 接口 + // --------------------------------------------------------------------------------------- + workflowV4 := router.Group("v4") + { + workflowV4.POST("", CreateWorkflowV4) + workflowV4.GET("", ListWorkflowV4) + workflowV4.POST("/lint", LintWorkflowV4) + workflowV4.GET("/name/:name", FindWorkflowV4) + workflowV4.PUT("/:name", UpdateWorkflowV4) + workflowV4.DELETE("/:name", DeleteWorkflowV4) + workflowV4.GET("/preset/:name", GetWorkflowV4Preset) + } + + // --------------------------------------------------------------------------------------- + // workflow v4 任务接口 + // --------------------------------------------------------------------------------------- + taskV4 := router.Group("v4/workflowtask") + { + taskV4.POST("", CreateWorkflowTaskV4) + taskV4.GET("", ListWorkflowTaskV4) + taskV4.GET("/workflow/:workflowName/task/:taskID", GetWorkflowTaskV4) + taskV4.DELETE("/workflow/:workflowName/task/:taskID", CancelWorkflowTaskV4) + taskV4.GET("/clone/workflow/:workflowName/task/:taskID", CloneWorkflowTaskV4) + taskV4.POST("/approve", ApproveStage) + } + + // --------------------------------------------------------------------------------------- + // plugin repo 接口 + // --------------------------------------------------------------------------------------- + plugin := router.Group("plugin") + { + plugin.GET("/template", ListPluginTemplates) + } + + bundles := router.Group("bundle-resources") + { + bundles.GET("", GetBundleResources) + } } diff --git a/pkg/microservice/aslan/core/workflow/handler/sse.go b/pkg/microservice/aslan/core/workflow/handler/sse.go index a83214fa900888a5d91f9230696b76f61f653b47..6257f294c5d261cdd649afe53b3a15a6e8d628cb 100644 --- a/pkg/microservice/aslan/core/workflow/handler/sse.go +++ b/pkg/microservice/aslan/core/workflow/handler/sse.go @@ -21,14 +21,17 @@ import ( "strconv" "time" - "k8s.io/apimachinery/pkg/util/wait" - "github.com/gin-gonic/gin" + "github.com/jinzhu/copier" + "k8s.io/apimachinery/pkg/util/wait" "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/workflowcontroller" + "github.com/koderover/zadig/pkg/microservice/aslan/core/delivery/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/types/dto" ) func GetPipelineTaskSSE(c *gin.Context) { @@ -46,14 +49,14 @@ func GetPipelineTaskSSE(c *gin.Context) { err := wait.PollImmediateUntil(time.Second, func() (bool, error) { res, err := workflow.GetPipelineTaskV2(taskID, c.Param("name"), config.SingleType, ctx.Logger) if err != nil { - ctx.Logger.Errorf("[%s] GetPipelineTaskSSE error: %v", ctx.Username, err) + ctx.Logger.Errorf("[%s] GetPipelineTaskSSE error: %v", ctx.UserName, err) return false, err } msgChan <- res if time.Since(startTime).Minutes() == float64(60) { - ctx.Logger.Warnf("[%s] Query GetPipelineTaskSSE API over 60 minutes", ctx.Username) + ctx.Logger.Warnf("[%s] Query GetPipelineTaskSSE API over 60 minutes", ctx.UserName) } return false, nil @@ -74,7 +77,7 @@ func RunningPipelineTasksSSE(c *gin.Context) { msgChan <- workflow.RunningPipelineTasks() if time.Since(startTime).Minutes() == float64(60) { - ctx.Logger.Warnf("[%s] Query RunningPipelineTasksSSE API over 60 minutes", ctx.Username) + ctx.Logger.Warnf("[%s] Query RunningPipelineTasksSSE API over 60 minutes", ctx.UserName) } }, time.Second) }, ctx.Logger) @@ -89,7 +92,37 @@ func PendingPipelineTasksSSE(c *gin.Context) { msgChan <- workflow.PendingPipelineTasks() if time.Since(startTime).Minutes() == float64(60) { - ctx.Logger.Warnf("[%s] Query PendingPipelineTasksSSE API over 60 minutes", ctx.Username) + ctx.Logger.Warnf("[%s] Query PendingPipelineTasksSSE API over 60 minutes", ctx.UserName) + } + }, time.Second) + }, ctx.Logger) +} + +func RunningWorkflowTasksSSE(c *gin.Context) { + ctx := internalhandler.NewContext(c) + + internalhandler.Stream(c, func(ctx1 context.Context, msgChan chan interface{}) { + startTime := time.Now() + wait.NonSlidingUntilWithContext(ctx1, func(_ context.Context) { + msgChan <- workflowcontroller.RunningTasks() + + if time.Since(startTime).Minutes() == float64(60) { + ctx.Logger.Warnf("[%s] Query RunningPipelineTasksSSE API over 60 minutes", ctx.UserName) + } + }, time.Second) + }, ctx.Logger) +} + +func PendingWorkflowTasksSSE(c *gin.Context) { + ctx := internalhandler.NewContext(c) + + internalhandler.Stream(c, func(ctx1 context.Context, msgChan chan interface{}) { + startTime := time.Now() + wait.NonSlidingUntilWithContext(ctx1, func(_ context.Context) { + msgChan <- workflowcontroller.PendingTasks() + + if time.Since(startTime).Minutes() == float64(60) { + ctx.Logger.Warnf("[%s] Query PendingPipelineTasksSSE API over 60 minutes", ctx.UserName) } }, time.Second) }, ctx.Logger) @@ -116,14 +149,67 @@ func GetWorkflowTaskSSE(c *gin.Context) { err := wait.PollImmediateUntil(time.Second, func() (bool, error) { res, err := workflow.GetPipelineTaskV2(taskID, c.Param("name"), workflowTypeString, ctx.Logger) if err != nil { - ctx.Logger.Errorf("[%s] GetPipelineTaskSSE error: %v", ctx.Username, err) + ctx.Logger.Errorf("[%s] GetPipelineTaskSSE error: %v", ctx.UserName, err) + return false, err + } + releases, err := service.ListDeliveryVersion(&service.ListDeliveryVersionArgs{ + TaskId: int(res.TaskID), + ServiceName: res.ServiceName, + ProjectName: res.ProductName, + WorkflowName: res.PipelineName, + }, ctx.Logger) + + re := []dto.Release{} + for _, v := range releases { + re = append(re, dto.Release{ + ID: v.VersionInfo.ID, + Version: v.VersionInfo.Version, + }) + } + var task dto.Task + if err := copier.Copy(&task, res); err != nil { + return false, err + } + task.Releases = re + + msgChan <- task + + if time.Since(startTime).Minutes() == float64(60) { + ctx.Logger.Warnf("[%s] Query GetPipelineTaskSSE API over 60 minutes", ctx.UserName) + } + + return false, nil + }, ctx1.Done()) + + if err != nil && err != wait.ErrWaitTimeout { + ctx.Logger.Error(err) + } + }, ctx.Logger) +} + +func GetWorkflowTaskV3SSE(c *gin.Context) { + ctx := internalhandler.NewContext(c) + + taskID, err := strconv.ParseInt(c.Param("id"), 10, 64) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("Invalid id Args") + internalhandler.JSONResponse(c, ctx) + return + } + + internalhandler.Stream(c, func(ctx1 context.Context, msgChan chan interface{}) { + startTime := time.Now() + err := wait.PollImmediateUntil(time.Second, func() (bool, error) { + res, err := workflow.GetWorkflowTaskV3(taskID, c.Param("name"), config.WorkflowTypeV3, ctx.Logger) + if err != nil { + ctx.Logger.Errorf("[%s] GetWorkflowTaskV3SSE error: %s", ctx.UserName, err) return false, err } msgChan <- res if time.Since(startTime).Minutes() == float64(60) { - ctx.Logger.Warnf("[%s] Query GetPipelineTaskSSE API over 60 minutes", ctx.Username) + ctx.Logger.Warnf("[%s] Query GetWorkflowTaskV3SSE API over 60 minutes", ctx.UserName) } return false, nil diff --git a/pkg/microservice/aslan/core/workflow/handler/webhook.go b/pkg/microservice/aslan/core/workflow/handler/webhook.go index 9179049c26f02015977a4307d0f4b1034e199e35..2a2cdab9bdb528a644d6f214ca47f133f0e8be4c 100644 --- a/pkg/microservice/aslan/core/workflow/handler/webhook.go +++ b/pkg/microservice/aslan/core/workflow/handler/webhook.go @@ -28,7 +28,7 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/webhook" internalhandler "github.com/koderover/zadig/pkg/shared/handler" "github.com/koderover/zadig/pkg/tool/codehub" - "github.com/koderover/zadig/pkg/tool/ilyshin" + "github.com/koderover/zadig/pkg/tool/gitee" ) // @Router /workflow/webhook [POST] @@ -51,8 +51,8 @@ func ProcessWebHook(c *gin.Context) { ctx.Err = webhook.ProcessGitlabHook(payload, c.Request, ctx.RequestID, ctx.Logger) } else if codehub.HookEventType(c.Request) != "" { ctx.Err = webhook.ProcessCodehubHook(payload, c.Request, ctx.RequestID, ctx.Logger) - } else if ilyshin.HookEventType(c.Request) != "" { - ctx.Err = webhook.ProcessIlyshinHook(payload, c.Request, ctx.RequestID, ctx.Logger) + } else if gitee.HookEventType(c.Request) != "" { + ctx.Err = webhook.ProcessGiteeHook(payload, c.Request, ctx.RequestID, ctx.Logger) } else { ctx.Err = webhook.ProcessGerritHook(payload, c.Request, ctx.RequestID, ctx.Logger) } @@ -75,6 +75,17 @@ func processGithub(payload []byte, req *http.Request, requestID string, log *zap log.Errorf("error happens to trigger workflow %v", err) errs = multierror.Append(errs, err) } - + //测试管理webhook + err = webhook.ProcessGithubWebHookForTest(payload, req, requestID, log) + if err != nil { + log.Errorf("error happens to trigger ProcessGithubWebHookForTest %v", err) + errs = multierror.Append(errs, err) + } + // webhooks for scanning task + err = webhook.ProcessGithubWebhookForScanning(payload, req, requestID, log) + if err != nil { + log.Errorf("error happens to trigger Scanning for github %v", err) + errs = multierror.Append(errs, err) + } return errs.ErrorOrNil() } diff --git a/pkg/microservice/aslan/core/workflow/handler/workflow.go b/pkg/microservice/aslan/core/workflow/handler/workflow.go index 729fcf3f369f33503f60ec8ad34aae52df83fc4a..3f15d9c52ebbab6e7ff4d3f80cefe5a26b271898 100644 --- a/pkg/microservice/aslan/core/workflow/handler/workflow.go +++ b/pkg/microservice/aslan/core/workflow/handler/workflow.go @@ -78,15 +78,15 @@ func CreateWorkflow(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("CreateWorkflow json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, args.ProductTmplName, "新增", "工作流", args.Name, string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, args.ProductTmplName, "新增", "工作流", args.Name, string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) if err := c.ShouldBindWith(&args, binding.JSON); err != nil { ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) return } - args.UpdateBy = ctx.Username - args.CreateBy = ctx.Username + args.UpdateBy = ctx.UserName + args.CreateBy = ctx.UserName ctx.Err = workflow.CreateWorkflow(args, ctx.Logger) } @@ -103,14 +103,14 @@ func UpdateWorkflow(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("UpdateWorkflow json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, args.ProductTmplName, "更新", "工作流", args.Name, string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, args.ProductTmplName, "更新", "工作流", args.Name, string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) if err := c.ShouldBindWith(&args, binding.JSON); err != nil { ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) return } - args.UpdateBy = ctx.Username + args.UpdateBy = ctx.UserName ctx.Err = workflow.UpdateWorkflow(args, ctx.Logger) } @@ -118,14 +118,30 @@ func ListWorkflows(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = workflow.ListWorkflows(c.Query("type"), ctx.User.ID, ctx.Logger) + projects := c.QueryArray("projects") + projectName := c.Query("projectName") + if projectName != "" && len(projects) > 0 { + ctx.Err = e.ErrInvalidParam.AddDesc("projects and projectName can not be set together") + return + } + if projectName != "" { + projects = []string{c.Query("projectName")} + } + + workflowNames, found := internalhandler.GetResourcesInHeader(c) + if found && len(workflowNames) == 0 { + ctx.Resp = []*workflow.Workflow{} + return + } + + ctx.Resp, ctx.Err = workflow.ListWorkflows(projects, ctx.UserID, workflowNames, ctx.Logger) } -func ListAllWorkflows(c *gin.Context) { +func ListTestWorkflows(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = workflow.ListAllWorkflows(c.Param("testName"), ctx.User.ID, ctx.User.IsSuperUser, ctx.Logger) + ctx.Resp, ctx.Err = workflow.ListTestWorkflows(c.Param("testName"), c.QueryArray("projects"), ctx.Logger) } // FindWorkflow find a workflow @@ -140,7 +156,7 @@ func DeleteWorkflow(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - internalhandler.InsertOperationLog(c, ctx.Username, c.GetString("productName"), "删除", "工作流", c.Param("name"), "", ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, c.GetString("productName"), "删除", "工作流", c.Param("name"), "", ctx.Logger) ctx.Err = commonservice.DeleteWorkflow(c.Param("name"), ctx.RequestID, false, ctx.Logger) } @@ -155,5 +171,5 @@ func CopyWorkflow(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Err = workflow.CopyWorkflow(c.Param("old"), c.Param("new"), ctx.Username, ctx.Logger) + ctx.Err = workflow.CopyWorkflow(c.Param("old"), c.Param("new"), ctx.UserName, ctx.Logger) } diff --git a/pkg/microservice/aslan/core/workflow/handler/workflow_task.go b/pkg/microservice/aslan/core/workflow/handler/workflow_task.go index 4ec372617590868d3fdb861bb9da95e5dad2b3e4..0384ba4e8dfda7de8937046619078fc75b9c4ae8 100644 --- a/pkg/microservice/aslan/core/workflow/handler/workflow_task.go +++ b/pkg/microservice/aslan/core/workflow/handler/workflow_task.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The KodeRover Authors. +Copyright 2022 The KodeRover Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -21,53 +21,24 @@ import ( "encoding/json" "io/ioutil" "strconv" + "strings" "github.com/gin-gonic/gin" "github.com/gin-gonic/gin/binding" + "github.com/jinzhu/copier" "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/delivery/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" "github.com/koderover/zadig/pkg/setting" internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types/dto" ) -func GetWorkflowTaskProductName(c *gin.Context) { - args := new(commonmodels.WorkflowTaskArgs) - data, err := c.GetRawData() - if err != nil { - log.Errorf("c.GetRawData() err : %v", err) - return - } - if err = json.Unmarshal(data, args); err != nil { - log.Errorf("json.Unmarshal err : %v", err) - return - } - c.Set("productName", args.ProductTmplName) - c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) - c.Next() -} - -func GetWorkflowTaskProductNameByTask(c *gin.Context) { - ctx := internalhandler.NewContext(c) - taskID, err := strconv.ParseInt(c.Param("id"), 10, 64) - if err != nil { - ctx.Err = e.ErrInvalidParam.AddDesc("invalid task id") - return - } - - pipelineTask, err := workflow.GetPipelineTaskV2(taskID, c.Param("name"), config.WorkflowType, ctx.Logger) - if err != nil { - log.Errorf("GetPipelineTaskV2 err : %v", err) - return - } - c.Set("productName", pipelineTask.WorkflowArgs.ProductTmplName) - c.Next() -} - // GetWorkflowArgs find workflow args func GetWorkflowArgs(c *gin.Context) { ctx := internalhandler.NewContext(c) @@ -97,7 +68,7 @@ func CreateWorkflowTask(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("CreateWorkflowTask json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, args.ProductTmplName, "新增", "工作流-task", args.WorkflowName, string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, args.ProductTmplName, "新增", "工作流-task", args.WorkflowName, string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) if err := c.ShouldBindWith(&args, binding.JSON); err != nil { @@ -105,15 +76,15 @@ func CreateWorkflowTask(c *gin.Context) { return } - if args.WorklowTaskCreator != setting.CronTaskCreator && args.WorklowTaskCreator != setting.WebhookTaskCreator { - args.WorklowTaskCreator = ctx.Username + if args.WorkflowTaskCreator != setting.CronTaskCreator && args.WorkflowTaskCreator != setting.WebhookTaskCreator { + args.WorkflowTaskCreator = ctx.UserName } - ctx.Resp, ctx.Err = workflow.CreateWorkflowTask(args, args.WorklowTaskCreator, ctx.User.ID, ctx.User.IsSuperUser, ctx.Logger) + ctx.Resp, ctx.Err = workflow.CreateWorkflowTask(args, args.WorkflowTaskCreator, ctx.Logger) // 发送通知 if ctx.Err != nil { - commonservice.SendFailedTaskMessage(ctx.Username, args.ProductTmplName, args.WorkflowName, ctx.RequestID, config.WorkflowType, ctx.Err, ctx.Logger) + commonservice.SendFailedTaskMessage(ctx.UserName, args.ProductTmplName, args.WorkflowName, ctx.RequestID, config.WorkflowType, ctx.Err, ctx.Logger) } } @@ -130,7 +101,7 @@ func CreateArtifactWorkflowTask(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("CreateArtifactWorkflowTask json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, args.ProductTmplName, "新增", "工作流-task", args.WorkflowName, string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, args.ProductTmplName, "新增", "工作流-task", args.WorkflowName, string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) if err := c.ShouldBindWith(&args, binding.JSON); err != nil { @@ -138,11 +109,11 @@ func CreateArtifactWorkflowTask(c *gin.Context) { return } - if args.WorklowTaskCreator != setting.CronTaskCreator && args.WorklowTaskCreator != setting.WebhookTaskCreator { - args.WorklowTaskCreator = ctx.Username + if args.WorkflowTaskCreator != setting.CronTaskCreator && args.WorkflowTaskCreator != setting.WebhookTaskCreator { + args.WorkflowTaskCreator = ctx.UserName } - ctx.Resp, ctx.Err = workflow.CreateArtifactWorkflowTask(args, args.WorklowTaskCreator, ctx.User.ID, ctx.User.IsSuperUser, ctx.Logger) + ctx.Resp, ctx.Err = workflow.CreateArtifactWorkflowTask(args, args.WorkflowTaskCreator, ctx.Logger) } // ListWorkflowTasksResult workflowtask分页信息 @@ -165,7 +136,19 @@ func ListWorkflowTasksResult(c *gin.Context) { if workflowType == string(config.TestType) { workflowTypeString = config.TestType } - ctx.Resp, ctx.Err = workflow.ListPipelineTasksV2Result(c.Param("name"), workflowTypeString, maxResult, startAt, ctx.Logger) + filters := c.Query("filters") + filtersList := strings.Split(filters, ",") + ctx.Resp, ctx.Err = workflow.ListPipelineTasksV2Result(c.Param("name"), workflowTypeString, c.Query("queryType"), filtersList, maxResult, startAt, ctx.Logger) +} + +func GetFiltersPipeline(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + if c.Query("workflowType") != string(config.WorkflowType) { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid workflowType") + return + } + ctx.Resp, ctx.Err = workflow.GetFiltersPipelineTaskV2(c.Query("projectName"), c.Param("name"), c.Query("queryType"), config.WorkflowType, ctx.Logger) } func GetWorkflowTask(c *gin.Context) { @@ -182,13 +165,44 @@ func GetWorkflowTask(c *gin.Context) { if workflowType == string(config.TestType) { workflowTypeString = config.TestType } - ctx.Resp, ctx.Err = workflow.GetPipelineTaskV2(taskID, c.Param("name"), workflowTypeString, ctx.Logger) + task, err := workflow.GetPipelineTaskV2(taskID, c.Param("name"), workflowTypeString, ctx.Logger) + if err != nil { + ctx.Err = err + return + } + releases, err := service.ListDeliveryVersion(&service.ListDeliveryVersionArgs{ + TaskId: int(task.TaskID), + ServiceName: task.ServiceName, + ProjectName: task.ProductName, + WorkflowName: task.PipelineName, + }, ctx.Logger) + if err != nil { + ctx.Err = err + return + } + + var toReleases []dto.Release + for _, v := range releases { + toReleases = append(toReleases, dto.Release{ + ID: v.VersionInfo.ID, + Version: v.VersionInfo.Version, + }) + } + var toTask dto.Task + if err := copier.Copy(&toTask, task); err != nil { + ctx.Err = err + return + } + toTask.Releases = toReleases + ctx.Resp = toTask + ctx.Err = err + return } func RestartWorkflowTask(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - internalhandler.InsertOperationLog(c, ctx.Username, c.GetString("productName"), "重启", "工作流-task", c.Param("name"), "", ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, c.Query("projectName"), "重启", "工作流-task", c.Param("name"), "", ctx.Logger) taskID, err := strconv.ParseInt(c.Param("id"), 10, 64) if err != nil { @@ -196,18 +210,31 @@ func RestartWorkflowTask(c *gin.Context) { return } - ctx.Err = workflow.RestartPipelineTaskV2(ctx.Username, taskID, c.Param("name"), config.WorkflowType, ctx.Logger) + ctx.Err = workflow.RestartPipelineTaskV2(ctx.UserName, taskID, c.Param("name"), config.WorkflowType, ctx.Logger) } func CancelWorkflowTaskV2(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - internalhandler.InsertOperationLog(c, ctx.Username, c.GetString("productName"), "取消", "工作流-task", c.Param("name"), "", ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, c.Query("projectName"), "取消", "工作流-task", c.Param("name"), "", ctx.Logger) + + taskID, err := strconv.ParseInt(c.Param("id"), 10, 64) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid task id") + return + } + ctx.Err = commonservice.CancelTaskV2(ctx.UserName, c.Param("name"), taskID, config.WorkflowType, ctx.RequestID, ctx.Logger) +} + +func GetWorkflowTaskCallback(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() taskID, err := strconv.ParseInt(c.Param("id"), 10, 64) if err != nil { ctx.Err = e.ErrInvalidParam.AddDesc("invalid task id") return } - ctx.Err = commonservice.CancelTaskV2(ctx.Username, c.Param("name"), taskID, config.WorkflowType, ctx.RequestID, ctx.Logger) + + ctx.Resp, ctx.Err = commonservice.GetWorkflowTaskCallback(taskID, c.Param("name")) } diff --git a/pkg/microservice/aslan/core/workflow/handler/workflow_task_v4.go b/pkg/microservice/aslan/core/workflow/handler/workflow_task_v4.go new file mode 100644 index 0000000000000000000000000000000000000000..938ecf32e7d98c6e698aa326f0da242492b8935a --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/handler/workflow_task_v4.go @@ -0,0 +1,150 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "bytes" + "encoding/json" + "io/ioutil" + "strconv" + + "github.com/gin-gonic/gin" + + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/log" +) + +type listWorkflowTaskV4Query struct { + PageSize int64 `json:"page_size" form:"page_size,default=20"` + PageNum int64 `json:"page_num" form:"page_num,default=1"` + WorkflowName string `json:"workflow_name" form:"workflow_name"` +} + +type listWorkflowTaskV4Resp struct { + WorkflowList []*commonmodels.WorkflowTask `json:"workflow_list"` + Total int64 `json:"total"` +} + +type ApproveRequest struct { + StageName string `json:"stage_name"` + WorkflowName string `json:"workflow_name"` + TaskID int64 `json:"task_id"` + Approve bool `json:"approve"` + Comment string `json:"comment"` +} + +func CreateWorkflowTaskV4(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(commonmodels.WorkflowV4) + data, err := c.GetRawData() + if err != nil { + log.Errorf("CreateWorkflowTaskv4 c.GetRawData() err : %s", err) + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("CreateWorkflowTaskv4 json.Unmarshal err : %s", err) + } + + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + + if err := c.ShouldBindJSON(&args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } + ctx.Resp, ctx.Err = workflow.CreateWorkflowTaskV4(ctx.UserName, args, ctx.Logger) +} + +func ListWorkflowTaskV4(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + args := &listWorkflowTaskV4Query{} + if err := c.ShouldBindQuery(args); err != nil { + ctx.Err = err + return + } + + taskList, total, err := workflow.ListWorkflowTaskV4(args.WorkflowName, args.PageNum, args.PageSize, ctx.Logger) + resp := listWorkflowTaskV4Resp{ + WorkflowList: taskList, + Total: total, + } + ctx.Resp = resp + ctx.Err = err +} + +func GetWorkflowTaskV4(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + taskID, err := strconv.ParseInt(c.Param("taskID"), 10, 64) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid task id") + return + } + ctx.Resp, ctx.Err = workflow.GetWorkflowTaskV4(c.Param("workflowName"), taskID, ctx.Logger) +} + +func CancelWorkflowTaskV4(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + taskID, err := strconv.ParseInt(c.Param("taskID"), 10, 64) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid task id") + return + } + ctx.Err = workflow.CancelWorkflowTaskV4(ctx.UserName, c.Param("workflowName"), taskID, ctx.Logger) +} + +func CloneWorkflowTaskV4(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + taskID, err := strconv.ParseInt(c.Param("taskID"), 10, 64) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid task id") + return + } + ctx.Resp, ctx.Err = workflow.CloneWorkflowTaskV4(c.Param("workflowName"), taskID, ctx.Logger) +} + +func ApproveStage(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + args := &ApproveRequest{} + + data, err := c.GetRawData() + if err != nil { + log.Errorf("ApproveStage c.GetRawData() err : %s", err) + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("ApproveStage json.Unmarshal err : %s", err) + } + + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + + if err := c.ShouldBindJSON(&args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } + + ctx.Err = workflow.ApproveStage(args.WorkflowName, args.StageName, ctx.UserName, ctx.UserID, args.Comment, args.TaskID, args.Approve, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/workflow/handler/workflow_v3.go b/pkg/microservice/aslan/core/workflow/handler/workflow_v3.go new file mode 100644 index 0000000000000000000000000000000000000000..129395a252ce7cd5577bea668a5863c2d9ab4999 --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/handler/workflow_v3.go @@ -0,0 +1,136 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "bytes" + "encoding/json" + "io/ioutil" + + "github.com/gin-gonic/gin" + + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + workflowservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/log" +) + +type createWorkflowResp struct { + ID string `json:"id"` +} + +func CreateWorkflowV3(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + req := &commonmodels.WorkflowV3{} + data, err := c.GetRawData() + if err != nil { + log.Errorf("CreateWorkflowV3 err: %s", err) + } + if err = json.Unmarshal(data, req); err != nil { + log.Errorf("CreateWorkflow unmarshal json err: %s", err) + } + internalhandler.InsertOperationLog(c, ctx.UserName, req.ProjectName, "新增", "工作流V3", req.Name, string(data), ctx.Logger) + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + + if err := c.ShouldBindJSON(&req); err != nil { + ctx.Err = errors.ErrInvalidParam.AddDesc(err.Error()) + return + } + workflowID, err := workflowservice.CreateWorkflowV3(ctx.UserName, req, ctx.Logger) + resp := createWorkflowResp{ID: workflowID} + ctx.Resp = resp + ctx.Err = err +} + +type listWorkflowV3Query struct { + PageSize int64 `json:"page_size" form:"page_size,default=20"` + PageNum int64 `json:"page_num" form:"page_num,default=1"` + ProjectName string `json:"project_name" form:"project_name"` +} + +type listWorkflowV3Resp struct { + WorkflowList []*workflowservice.WorkflowV3Brief `json:"workflow_list"` + Total int64 `json:"total"` +} + +func ListWorkflowsV3(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + // Query Verification + args := &listWorkflowV3Query{} + if err := c.ShouldBindQuery(args); err != nil { + ctx.Err = err + return + } + + workflowList, total, err := workflowservice.ListWorkflowsV3(args.ProjectName, args.PageNum, args.PageSize, ctx.Logger) + resp := listWorkflowV3Resp{ + WorkflowList: workflowList, + Total: total, + } + ctx.Resp = resp + ctx.Err = err +} + +func GetWorkflowV3Detail(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = workflowservice.GetWorkflowV3Detail(c.Param("id"), ctx.Logger) +} + +func UpdateWorkflowV3(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + req := &commonmodels.WorkflowV3{} + + data, err := c.GetRawData() + if err != nil { + log.Errorf("UpdateWorkflowV3 err: %s", err) + } + if err = json.Unmarshal(data, req); err != nil { + log.Errorf("UpdateWorkflowV3 unmarshal json err: %s", err) + } + internalhandler.InsertOperationLog(c, ctx.UserName, req.ProjectName, "修改", "工作流V3", req.Name, string(data), ctx.Logger) + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + + if err := c.ShouldBindJSON(&req); err != nil { + ctx.Err = errors.ErrInvalidParam.AddDesc(err.Error()) + return + } + + ctx.Err = workflowservice.UpdateWorkflowV3(c.Param("id"), ctx.UserName, req, ctx.Logger) +} + +func DeleteWorkflowV3(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Err = workflowservice.DeleteWorkflowV3(c.Param("id"), ctx.Logger) +} + +func GetWorkflowV3Args(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = workflowservice.GetWorkflowV3Args(c.Param("id"), ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/workflow/handler/workflow_v3_task.go b/pkg/microservice/aslan/core/workflow/handler/workflow_v3_task.go new file mode 100644 index 0000000000000000000000000000000000000000..13d2e495bcef9983422f8255e524b194b1bd8c1f --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/handler/workflow_v3_task.go @@ -0,0 +1,149 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "bytes" + "encoding/json" + "io/ioutil" + "strconv" + + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/log" +) + +func CreateWorkflowTaskV3(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(commonmodels.WorkflowV3Args) + data, err := c.GetRawData() + if err != nil { + log.Errorf("CreateWorkflowTaskV3 c.GetRawData() err : %s", err) + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("CreateWorkflowTaskV3 json.Unmarshal err : %s", err) + } + internalhandler.InsertOperationLog(c, ctx.UserName, args.ProjectName, "创建", "工作流v3-task", args.Name, string(data), ctx.Logger) + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + + if err := c.ShouldBindJSON(&args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } + + ctx.Resp, ctx.Err = workflow.CreateWorkflowTaskV3(args, ctx.UserName, ctx.RequestID, ctx.Logger) +} + +func RestartWorkflowTaskV3(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + internalhandler.InsertOperationLog(c, ctx.UserName, "", "重启", "工作流taskV3", c.Param("name"), "", ctx.Logger) + + taskID, err := strconv.ParseInt(c.Param("id"), 10, 64) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid task id") + return + } + + ctx.Err = workflow.RestartWorkflowTaskV3(ctx.UserName, taskID, c.Param("name"), config.WorkflowTypeV3, ctx.Logger) +} + +func CancelWorkflowTaskV3(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + internalhandler.InsertOperationLog(c, ctx.UserName, c.Query("projectName"), "取消", "工作流taskV3", c.Param("name"), "", ctx.Logger) + + taskID, err := strconv.ParseInt(c.Param("id"), 10, 64) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid task id") + return + } + ctx.Err = commonservice.CancelWorkflowTaskV3(ctx.UserName, c.Param("name"), taskID, config.WorkflowTypeV3, ctx.RequestID, ctx.Logger) +} + +// ListWorkflowV3TasksResult workflowtask分页信息 +func ListWorkflowV3TasksResult(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + maxResult, err := strconv.Atoi(c.Param("max")) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid max result number") + return + } + startAt, err := strconv.Atoi(c.Param("start")) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid start at number") + return + } + + ctx.Resp, ctx.Err = workflow.ListWorkflowTasksV3Result(c.Param("name"), config.WorkflowTypeV3, maxResult, startAt, ctx.Logger) +} + +func GetWorkflowTaskV3(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + taskID, err := strconv.ParseInt(c.Param("id"), 10, 64) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid task id") + return + } + + ctx.Resp, ctx.Err = workflow.GetWorkflowTaskV3(taskID, c.Param("name"), config.WorkflowTypeV3, ctx.Logger) +} + +type WebhookPayload struct { + EventName string `json:"event_name"` + ProjectName string `json:"project_name"` + TaskName string `json:"task_name"` + TaskID int64 `json:"task_id"` + TaskOutput []*TaskOutput `json:"task_output"` + TaskEnvs []*KeyVal `json:"task_envs"` +} + +type TaskOutput struct { + Type string `json:"type"` + Value string `json:"value"` +} + +type KeyVal struct { + Key string `bson:"key" json:"key"` + Value string `bson:"value" json:"value"` + IsCredential bool `bson:"is_credential" json:"is_credential"` +} + +func GetWorkflowTaskV3Callback(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + taskID, err := strconv.ParseInt(c.Param("id"), 10, 64) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid task id") + return + } + + ctx.Resp, ctx.Err = workflow.GetWorkflowTaskV3Callback(taskID, c.Param("name"), ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/workflow/handler/workflow_v4.go b/pkg/microservice/aslan/core/workflow/handler/workflow_v4.go new file mode 100644 index 0000000000000000000000000000000000000000..25832dc20079495097a4e451ab33952b59f09d9f --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/handler/workflow_v4.go @@ -0,0 +1,157 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "bytes" + "io/ioutil" + + "github.com/gin-gonic/gin" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/log" + "gopkg.in/yaml.v3" +) + +type listWorkflowV4Query struct { + PageSize int64 `json:"page_size" form:"page_size,default=20"` + PageNum int64 `json:"page_num" form:"page_num,default=1"` + Project string `json:"project" form:"project"` +} + +type listWorkflowV4Resp struct { + WorkflowList []*workflow.Workflow `json:"workflow_list"` + Total int64 `json:"total"` +} + +func CreateWorkflowV4(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(commonmodels.WorkflowV4) + data, err := c.GetRawData() + if err != nil { + log.Errorf("CreateWorkflowv4 c.GetRawData() err : %s", err) + } + if err = yaml.Unmarshal(data, args); err != nil { + log.Errorf("CreateWorkflowv4 json.Unmarshal err : %s", err) + } + + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + + if err := c.ShouldBindYAML(&args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } + + ctx.Err = workflow.CreateWorkflowV4(ctx.UserName, args, ctx.Logger) +} + +func LintWorkflowV4(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(commonmodels.WorkflowV4) + data, err := c.GetRawData() + if err != nil { + log.Errorf("CreateWorkflowv4 c.GetRawData() err : %s", err) + } + if err = yaml.Unmarshal(data, args); err != nil { + log.Errorf("CreateWorkflowv4 json.Unmarshal err : %s", err) + } + + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + + if err := c.ShouldBindYAML(&args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } + ctx.Err = workflow.LintWorkflowV4(args, ctx.Logger) +} + +func ListWorkflowV4(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + args := &listWorkflowV4Query{} + if err := c.ShouldBindQuery(args); err != nil { + ctx.Err = err + return + } + ignoreWorkflow := false + workflowNames, found := internalhandler.GetResourcesInHeader(c) + if found && len(workflowNames) == 0 { + ignoreWorkflow = true + } + + workflowList, err := workflow.ListWorkflowV4(args.Project, ctx.UserID, workflowNames, ignoreWorkflow, ctx.Logger) + resp := listWorkflowV4Resp{ + WorkflowList: workflowList, + Total: 0, + } + ctx.Resp = resp + ctx.Err = err +} + +func UpdateWorkflowV4(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(commonmodels.WorkflowV4) + data, err := c.GetRawData() + if err != nil { + log.Errorf("UpdateWorkflowV4 c.GetRawData() err : %s", err) + } + if err = yaml.Unmarshal(data, args); err != nil { + log.Errorf("UpdateWorkflowV4 json.Unmarshal err : %s", err) + } + + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + + if err := c.ShouldBindYAML(&args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } + + ctx.Err = workflow.UpdateWorkflowV4(c.Param("name"), ctx.UserName, args, ctx.Logger) +} + +func DeleteWorkflowV4(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Err = workflow.DeleteWorkflowV4(c.Param("name"), ctx.Logger) +} + +func FindWorkflowV4(c *gin.Context) { + ctx := internalhandler.NewContext(c) + resp, err := workflow.FindWorkflowV4(c.Query("encryptedKey"), c.Param("name"), ctx.Logger) + if err != nil { + c.JSON(e.ErrorMessage(err)) + c.Abort() + return + } + c.YAML(200, resp) +} + +func GetWorkflowV4Preset(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = workflow.GetWorkflowv4Preset(c.Query("encryptedKey"), c.Param("name"), ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/auto_cancel_task.go b/pkg/microservice/aslan/core/workflow/service/webhook/auto_cancel_task.go index d083fbf73635e80625faeaa367c5e04e3836a3f5..e0e338ad7fa9ea323d3b3c18054f2597ee52a4c6 100644 --- a/pkg/microservice/aslan/core/workflow/service/webhook/auto_cancel_task.go +++ b/pkg/microservice/aslan/core/workflow/service/webhook/auto_cancel_task.go @@ -89,14 +89,11 @@ func AutoCancelWorkflowTask(autoCancelOpt *AutoCancelOpt, task *task.Task, log * } for _, item := range workflow.HookCtl.Items { - if item.AutoCancel { - if item.MainRepo.RepoOwner != autoCancelOpt.MainRepo.RepoOwner || - item.MainRepo.RepoName != autoCancelOpt.MainRepo.RepoName || - item.MainRepo.Branch != autoCancelOpt.MainRepo.Branch || - item.MainRepo.Source != autoCancelOpt.MainRepo.Source { - continue - } - + autoCancel := item.AutoCancel + if item.IsYaml && autoCancelOpt.YamlHookPath == item.YamlPath { + autoCancel = autoCancelOpt.AutoCancel + } + if autoCancel { if err := commonservice.CancelTask(task.TaskCreator, task.PipelineName, task.TaskID, task.Type, task.ReqID, log); err != nil { log.Errorf("CancelRunningTask failed,task.TaskCreator:%s, task.PipelineName:%s, task.TaskID:%d, task.Type:%s, error: %v", task.TaskCreator, task.PipelineName, task.TaskID, task.Type, err) continue @@ -122,6 +119,7 @@ func AutoCancelTestTask(autoCancelOpt *AutoCancelOpt, task *task.Task, log *zap. } for _, item := range test.HookCtl.Items { + // TODO:testing support webhook as code trigger.yaml if item.AutoCancel { if item.MainRepo.RepoOwner != autoCancelOpt.MainRepo.RepoOwner || item.MainRepo.RepoName != autoCancelOpt.MainRepo.RepoName || diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/codehub.go b/pkg/microservice/aslan/core/workflow/service/webhook/codehub.go index 03faf1d8fbc4a921f22428e45256f1c04363f7b5..516729b13656ff6a9368b61e80a7f2a89f930a4b 100644 --- a/pkg/microservice/aslan/core/workflow/service/webhook/codehub.go +++ b/pkg/microservice/aslan/core/workflow/service/webhook/codehub.go @@ -20,7 +20,6 @@ import ( "errors" "fmt" "net/http" - "strings" "time" "github.com/hashicorp/go-multierror" @@ -107,7 +106,7 @@ func updateServiceTemplateByCodehubPushEvent(event *codehub.PushEvent, log *zap. } for _, fileName := range fileNames { - if strings.Contains(fileName, path) { + if subElem(path, fileName) { affected = true break } @@ -168,11 +167,7 @@ func SyncServiceTemplateFromCodehub(service *commonmodels.Service, log *zap.Suga log.Errorf("ensureServiceTmpl error: %+v", err) return e.ErrValidateTemplate.AddDesc(err.Error()) } - // 更新到数据库，revision+1 - if err := commonrepo.NewServiceColl().Create(service); err != nil { - log.Errorf("Failed to sync service %s from codehub path %s error: %v", service.ServiceName, service.SrcPath, err) - return e.ErrCreateTemplate.AddDesc(err.Error()) - } + log.Infof("End of sync service template %s from codehub path %s", service.ServiceName, service.SrcPath) return nil } diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/codehub_workflow_task.go b/pkg/microservice/aslan/core/workflow/service/webhook/codehub_workflow_task.go index 9a0eda6ddeff0d295883b5fabb51d4f29fe68b63..a1c2ed2c3a0620f295730970acd4f1baa1f1e967 100644 --- a/pkg/microservice/aslan/core/workflow/service/webhook/codehub_workflow_task.go +++ b/pkg/microservice/aslan/core/workflow/service/webhook/codehub_workflow_task.go @@ -30,7 +30,6 @@ import ( "github.com/koderover/zadig/pkg/setting" "github.com/koderover/zadig/pkg/tool/codehub" "github.com/koderover/zadig/pkg/types" - "github.com/koderover/zadig/pkg/types/permission" ) type codehubMergeEventMatcher struct { @@ -39,11 +38,12 @@ type codehubMergeEventMatcher struct { event *codehub.MergeEvent } -func (cmem *codehubMergeEventMatcher) Match(hookRepo commonmodels.MainHookRepo) (bool, error) { +func (cmem *codehubMergeEventMatcher) Match(hookRepo *commonmodels.MainHookRepo) (bool, error) { ev := cmem.event if (hookRepo.RepoOwner + "/" + hookRepo.RepoName) == ev.ObjectAttributes.Target.PathWithNamespace { if EventConfigured(hookRepo, config.HookEventPr) && (hookRepo.Branch == ev.ObjectAttributes.TargetBranch) { if ev.ObjectAttributes.State == "opened" { + hookRepo.Committer = ev.User.Username return true, nil } } @@ -52,7 +52,7 @@ func (cmem *codehubMergeEventMatcher) Match(hookRepo commonmodels.MainHookRepo) } func (cmem *codehubMergeEventMatcher) UpdateTaskArgs( - product *commonmodels.Product, args *commonmodels.WorkflowTaskArgs, hookRepo commonmodels.MainHookRepo, requestID string, + product *commonmodels.Product, args *commonmodels.WorkflowTaskArgs, hookRepo *commonmodels.MainHookRepo, requestID string, ) *commonmodels.WorkflowTaskArgs { factory := &workflowArgsFactory{ workflow: cmem.workflow, @@ -60,11 +60,12 @@ func (cmem *codehubMergeEventMatcher) UpdateTaskArgs( } args = factory.Update(product, args, &types.Repository{ - CodehostID: hookRepo.CodehostID, - RepoName: hookRepo.RepoName, - RepoOwner: hookRepo.RepoOwner, - Branch: hookRepo.Branch, - PR: cmem.event.ObjectAttributes.IID, + CodehostID: hookRepo.CodehostID, + RepoName: hookRepo.RepoName, + RepoOwner: hookRepo.RepoOwner, + RepoNamespace: hookRepo.GetRepoNamespace(), + Branch: hookRepo.Branch, + PR: cmem.event.ObjectAttributes.IID, }) return args @@ -76,10 +77,11 @@ type codehubPushEventMatcher struct { event *codehub.PushEvent } -func (cpem *codehubPushEventMatcher) Match(hookRepo commonmodels.MainHookRepo) (bool, error) { +func (cpem *codehubPushEventMatcher) Match(hookRepo *commonmodels.MainHookRepo) (bool, error) { ev := cpem.event if (hookRepo.RepoOwner + "/" + hookRepo.RepoName) == ev.Project.PathWithNamespace { if hookRepo.Branch == getBranchFromRef(ev.Ref) && EventConfigured(hookRepo, config.HookEventPush) { + hookRepo.Committer = ev.UserUsername return true, nil } } @@ -88,7 +90,7 @@ func (cpem *codehubPushEventMatcher) Match(hookRepo commonmodels.MainHookRepo) ( } func (cpem *codehubPushEventMatcher) UpdateTaskArgs( - product *commonmodels.Product, args *commonmodels.WorkflowTaskArgs, hookRepo commonmodels.MainHookRepo, requestID string, + product *commonmodels.Product, args *commonmodels.WorkflowTaskArgs, hookRepo *commonmodels.MainHookRepo, requestID string, ) *commonmodels.WorkflowTaskArgs { factory := &workflowArgsFactory{ workflow: cpem.workflow, @@ -96,10 +98,11 @@ func (cpem *codehubPushEventMatcher) UpdateTaskArgs( } factory.Update(product, args, &types.Repository{ - CodehostID: hookRepo.CodehostID, - RepoName: hookRepo.RepoName, - RepoOwner: hookRepo.RepoOwner, - Branch: hookRepo.Branch, + CodehostID: hookRepo.CodehostID, + RepoName: hookRepo.RepoName, + RepoOwner: hookRepo.RepoOwner, + RepoNamespace: hookRepo.GetRepoNamespace(), + Branch: hookRepo.Branch, }) return args @@ -198,9 +201,11 @@ func TriggerWorkflowByCodehubEvent(event interface{}, baseURI, requestID string, args.Source = setting.SourceFromCodeHub args.CodehostID = item.MainRepo.CodehostID args.RepoOwner = item.MainRepo.RepoOwner + args.RepoNamespace = item.MainRepo.GetRepoNamespace() args.RepoName = item.MainRepo.RepoName + args.Committer = item.MainRepo.Committer // 3. create task with args - if resp, err := workflowservice.CreateWorkflowTask(args, setting.WebhookTaskCreator, permission.AnonymousUserID, false, log); err != nil { + if resp, err := workflowservice.CreateWorkflowTask(args, setting.WebhookTaskCreator, log); err != nil { log.Errorf("failed to create workflow task when receive push event %v due to %v ", event, err) mErr = multierror.Append(mErr, err) } else { diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/gerrit.go b/pkg/microservice/aslan/core/workflow/service/webhook/gerrit.go index b3df30c12881a8a2dffc36cb72864ade0bc4752e..7486ee42870489335966aa109aeabee0b09765ae 100644 --- a/pkg/microservice/aslan/core/workflow/service/webhook/gerrit.go +++ b/pkg/microservice/aslan/core/workflow/service/webhook/gerrit.go @@ -20,23 +20,29 @@ import ( "encoding/json" "errors" "fmt" + "io/fs" "io/ioutil" "net/http" "os" "path" + "path/filepath" "strings" "github.com/hashicorp/go-multierror" + "github.com/otiai10/copy" "go.uber.org/zap" + systemConfig "github.com/koderover/zadig/pkg/config" "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/command" gerritservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/gerrit" + environmentservice "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/service/service" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/codehost" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/gerrit" "github.com/koderover/zadig/pkg/util" @@ -53,12 +59,7 @@ type gerritTypeEvent struct { } func ProcessGerritHook(payload []byte, req *http.Request, requestID string, log *zap.SugaredLogger) error { - baseURI := fmt.Sprintf( - "%s://%s", - req.Header.Get("X-Forwarded-Proto"), - req.Header.Get("X-Forwarded-Host"), - ) - + baseURI := systemConfig.SystemAddress() gerritTypeEventObj := new(gerritTypeEvent) if err := json.Unmarshal(payload, gerritTypeEventObj); err != nil { log.Errorf("processGerritHook json.Unmarshal err : %v", err) @@ -94,13 +95,13 @@ func updateServiceTemplateByGerritEvent(uri string, log *zap.SugaredLogger) erro log.Errorf("updateServiceTemplateByGerritEvent GetServiceTemplate err:%v", err) errs = multierror.Append(errs, err) } - detail, err := codehost.GetCodehostDetail(service.GerritCodeHostID) + detail, err := systemconfig.New().GetCodeHost(service.GerritCodeHostID) if err != nil { log.Errorf("updateServiceTemplateByGerritEvent GetCodehostDetail err:%v", err) errs = multierror.Append(errs, err) } newRepoName := fmt.Sprintf("%s-new", service.GerritRepoName) - err = command.RunGitCmds(detail, setting.GerritDefaultOwner, newRepoName, service.GerritBranchName, service.GerritRemoteName) + err = command.RunGitCmds(detail, setting.GerritDefaultOwner, setting.GerritDefaultOwner, newRepoName, service.GerritBranchName, service.GerritRemoteName) if err != nil { log.Errorf("updateServiceTemplateByGerritEvent runGitCmds err:%v", err) errs = multierror.Append(errs, err) @@ -114,13 +115,13 @@ func updateServiceTemplateByGerritEvent(uri string, log *zap.SugaredLogger) erro oldBase, err := GetGerritWorkspaceBasePath(service.GerritRepoName) if err != nil { errs = multierror.Append(errs, err) - err = command.RunGitCmds(detail, setting.GerritDefaultOwner, service.GerritRepoName, service.GerritBranchName, service.GerritRemoteName) + err = command.RunGitCmds(detail, setting.GerritDefaultOwner, setting.GerritDefaultOwner, service.GerritRepoName, service.GerritBranchName, service.GerritRemoteName) if err != nil { errs = multierror.Append(errs, err) } } - filePath, err := os.Stat(path.Join(newBase, service.GerritPath)) + filePath, err := os.Stat(path.Join(newBase, service.LoadPath)) if err != nil { errs = multierror.Append(errs, err) } @@ -128,44 +129,26 @@ func updateServiceTemplateByGerritEvent(uri string, log *zap.SugaredLogger) erro var newYamlContent string var oldYamlContent string if filePath.IsDir() { - newFileInfos, err := ioutil.ReadDir(path.Join(newBase, service.GerritPath)) - if err != nil { + if newFileContents, err := readAllFileContentUnderDir(path.Join(newBase, service.LoadPath)); err == nil { + newYamlContent = strings.Join(newFileContents, setting.YamlFileSeperator) + } else { errs = multierror.Append(errs, err) } - newFileContents := make([]string, 0) - for _, file := range newFileInfos { - if contentBytes, err := ioutil.ReadFile(path.Join(newBase, service.GerritPath, file.Name())); err == nil { - newFileContents = append(newFileContents, string(contentBytes)) - } else { - errs = multierror.Append(errs, err) - } - } - - newYamlContent = strings.Join(newFileContents, setting.YamlFileSeperator) - - oldFileInfos, err := ioutil.ReadDir(path.Join(oldBase, service.GerritPath)) - if err != nil { + if oldFileContents, err := readAllFileContentUnderDir(path.Join(oldBase, service.LoadPath)); err == nil { + oldYamlContent = strings.Join(oldFileContents, setting.YamlFileSeperator) + } else { errs = multierror.Append(errs, err) } - oldFileContents := make([]string, 0) - for _, file := range oldFileInfos { - if contentBytes, err := ioutil.ReadFile(path.Join(oldBase, service.GerritPath, file.Name())); err == nil { - oldFileContents = append(oldFileContents, string(contentBytes)) - } else { - errs = multierror.Append(errs, err) - } - } - oldYamlContent = strings.Join(oldFileContents, setting.YamlFileSeperator) } else { - if contentBytes, err := ioutil.ReadFile(path.Join(newBase, service.GerritPath)); err == nil { + if contentBytes, err := ioutil.ReadFile(path.Join(newBase, service.LoadPath)); err == nil { newYamlContent = string(contentBytes) } else { errs = multierror.Append(errs, err) } - if contentBytes, err := ioutil.ReadFile(path.Join(oldBase, service.GerritPath)); err == nil { + if contentBytes, err := ioutil.ReadFile(path.Join(oldBase, service.LoadPath)); err == nil { oldYamlContent = string(contentBytes) } else { errs = multierror.Append(errs, err) @@ -173,15 +156,14 @@ func updateServiceTemplateByGerritEvent(uri string, log *zap.SugaredLogger) erro } if strings.Compare(newYamlContent, oldYamlContent) != 0 { - log.Infof("Started to sync service template %s from gerrit %s", service.ServiceName, service.GerritPath) + log.Infof("Started to sync service template %s from gerrit %s", service.ServiceName, service.LoadPath) service.CreateBy = "system" service.Yaml = newYamlContent - err := SyncServiceTemplateFromGerrit(service, log) - if err != nil { + if err := SyncServiceTemplateFromGerrit(service, log); err != nil { errs = multierror.Append(errs, err) } } else { - log.Infof("Service template %s from gerrit %s is not affected, no sync", service.ServiceName, service.GerritPath) + log.Infof("Service template %s from gerrit %s is not affected, no sync", service.ServiceName, service.LoadPath) } } @@ -195,40 +177,59 @@ func GetGerritWorkspaceBasePath(repoName string) (string, error) { // GetGerritServiceTemplates Get all service templates maintained in gerrit func GetGerritServiceTemplates() ([]*commonmodels.Service, error) { opt := &commonrepo.ServiceListOption{ - Type: setting.K8SDeployType, Source: setting.SourceFromGerrit, } return commonrepo.NewServiceColl().ListMaxRevisions(opt) } -// SyncServiceTemplateFromGerrit Force to sync Service Template to latest commit and content, +// SyncServiceTemplateFromGerrit Force to sync Service Template to the latest commit and content, // Notes: if remains the same, quit sync; if updates, revision +1 func SyncServiceTemplateFromGerrit(service *commonmodels.Service, log *zap.SugaredLogger) error { if service.Source != setting.SourceFromGerrit { return fmt.Errorf("SyncServiceTemplateFromGerrit Service template is not from gerrit") } - //同步commit信息 + // Sync commit information if _, err := syncGerritLatestCommit(service); err != nil { log.Errorf("SyncServiceTemplateFromGerrit Sync change log from gerrit failed service %s, error: %v", service.ServiceName, err) return err } - // 在Ensure过程中会检查source，如果source为gerrit，则同步gerrit内容到service中 - if err := ensureServiceTmpl(setting.WebhookTaskCreator, service, log); err != nil { - log.Errorf("SyncServiceTemplateFromGerrit ensureServiceTmpl error: %+v", err) - return e.ErrValidateTemplate.AddDesc(err.Error()) + if service.Type == setting.K8SDeployType { + // During the Ensure process, the source will be checked. If the source is gerrit, the gerrit content will be synchronized to the service. + if err := ensureServiceTmpl(setting.WebhookTaskCreator, service, log); err != nil { + log.Errorf("SyncServiceTemplateFromGerrit ensureServiceTmpl error: %+v", err) + return e.ErrValidateTemplate.AddDesc(err.Error()) + } + // Update to database, revision+1 + if err := commonrepo.NewServiceColl().Create(service); err != nil { + log.Errorf("SyncServiceTemplateFromGerrit Failed to sync service %s from gerrit path %s error: %v", service.ServiceName, service.SrcPath, err) + return e.ErrCreateTemplate.AddDesc(err.Error()) + } + + return environmentservice.AutoDeployYamlServiceToEnvs(service.CreateBy, "", service, log) + } + // remove old repo dir + oldRepoDir := filepath.Join(config.S3StoragePath(), service.GerritRepoName) + os.RemoveAll(oldRepoDir) + // copy new repo data to old dir data + if err := os.MkdirAll(oldRepoDir, 0775); err != nil { + return err + } + + newRepoDir := filepath.Join(config.S3StoragePath(), service.GerritRepoName+"-new") + if copyErr := copy.Copy(newRepoDir, oldRepoDir); copyErr != nil { + return copyErr } - // 更新到数据库，revision+1 - if err := commonrepo.NewServiceColl().Create(service); err != nil { - log.Errorf("SyncServiceTemplateFromGerrit Failed to sync service %s from gerrit path %s error: %v", service.ServiceName, service.SrcPath, err) - return e.ErrCreateTemplate.AddDesc(err.Error()) + + if err := reloadServiceTmplFromGerrit(service, log); err != nil { + return err } log.Infof("End of sync service template %s from gerrit path %s", service.ServiceName, service.SrcPath) return nil } -func syncGerritLatestCommit(service *commonmodels.Service) (*codehost.Detail, error) { +func syncGerritLatestCommit(service *commonmodels.Service) (*systemconfig.CodeHost, error) { if service.GerritCodeHostID == 0 { return nil, fmt.Errorf("codehostId不能是空的") } @@ -238,12 +239,12 @@ func syncGerritLatestCommit(service *commonmodels.Service) (*codehost.Detail, er if service.GerritBranchName == "" { return nil, fmt.Errorf("branchName不能是空的") } - detail, err := codehost.GetCodehostDetail(service.GerritCodeHostID) + detail, err := systemconfig.New().GetCodeHost(service.GerritCodeHostID) if err != nil { return nil, err } - gerritCli := gerrit.NewClient(detail.Address, detail.OauthToken) + gerritCli := gerrit.NewClient(detail.Address, detail.AccessToken, config.ProxyHTTPSAddr(), detail.EnableProxy) commit, err := gerritCli.GetCommitByBranch(service.GerritRepoName, service.GerritBranchName) if err != nil { return detail, err @@ -284,26 +285,7 @@ func ensureServiceTmpl(userName string, args *commonmodels.Service, log *zap.Sug if err := setCurrentContainerImages(args); err != nil { return err } - //判断该服务组件是否存在，如果存在不让保存 - //if args.Revision == 0 { - // currentServiceContainerNames := make([]string, 0) - // for _, container := range args.Containers { - // currentServiceContainerNames = append(currentServiceContainerNames, container.Name) - // } - // if serviceTmpls, err := s.coll.ServiceTmpl.ListMaxRevisions(); err == nil { - // for _, serviceTmpl := range serviceTmpls { - // switch serviceTmpl.Type { - // case template.K8SDeployType: - // for _, container := range serviceTmpl.Containers { - // target := container.Name - // if utils.Contains(target, currentServiceContainerNames) { - // return fmt.Errorf("服务组件不能重复,项目 [%s] 服务 [%s] 已存在同名的服务组件 [%s]", serviceTmpl.ProductName, serviceTmpl.ServiceName, target) - // } - // } - // } - // } - // } - //} + log.Infof("find %d containers in service %s", len(args.Containers), args.ServiceName) } @@ -318,3 +300,47 @@ func ensureServiceTmpl(userName string, args *commonmodels.Service, log *zap.Sug return nil } + +func reloadServiceTmplFromGerrit(svc *commonmodels.Service, log *zap.SugaredLogger) error { + _, err := service.CreateOrUpdateHelmServiceFromRepo(svc.ProductName, &service.HelmServiceCreationArgs{ + HelmLoadSource: service.HelmLoadSource{ + Source: service.LoadFromGerrit, + }, + CreatedBy: svc.CreateBy, + CreateFrom: &service.CreateFromRepo{ + CodehostID: svc.CodehostID, + Owner: svc.RepoOwner, + Namespace: svc.GetRepoNamespace(), + Repo: svc.RepoName, + Branch: svc.BranchName, + Paths: []string{svc.LoadPath}, + }, + }, true, log) + return err +} + +// Get the contents of all files in a directory +func readAllFileContentUnderDir(localBase string) ([]string, error) { + fileTree := os.DirFS(localBase) + allFileContents := []string{} + err := fs.WalkDir(fileTree, ".", func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + + mode := entry.Type() + switch { + case mode.IsRegular(): + fileContents, err := fs.ReadFile(fileTree, path) + if err != nil { + return err + } + + allFileContents = append(allFileContents, string(fileContents)) + return nil + default: + return nil + } + }) + return allFileContents, err +} diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/gerrit_workflow_task.go b/pkg/microservice/aslan/core/workflow/service/webhook/gerrit_workflow_task.go index 7bbf9e2920b94dacfa22c305210da2ec7dc5f410..5261c714a223cdca7a829ae932899cc80ff32b99 100644 --- a/pkg/microservice/aslan/core/workflow/service/webhook/gerrit_workflow_task.go +++ b/pkg/microservice/aslan/core/workflow/service/webhook/gerrit_workflow_task.go @@ -34,10 +34,9 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/scmnotify" workflowservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/codehost" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" "github.com/koderover/zadig/pkg/tool/gerrit" "github.com/koderover/zadig/pkg/types" - "github.com/koderover/zadig/pkg/types/permission" ) type patchsetCreatedEvent struct { @@ -157,8 +156,8 @@ type SubmitterInfo struct { } type gerritEventMatcher interface { - Match(commonmodels.MainHookRepo) (bool, error) - UpdateTaskArgs(*commonmodels.Product, *commonmodels.WorkflowTaskArgs, commonmodels.MainHookRepo, string) *commonmodels.WorkflowTaskArgs + Match(*commonmodels.MainHookRepo) (bool, error) + UpdateTaskArgs(*commonmodels.Product, *commonmodels.WorkflowTaskArgs, *commonmodels.MainHookRepo, string) *commonmodels.WorkflowTaskArgs } type gerritChangeMergedEventMatcher struct { @@ -168,7 +167,7 @@ type gerritChangeMergedEventMatcher struct { Event *changeMergedEvent } -func (gruem *gerritChangeMergedEventMatcher) Match(hookRepo commonmodels.MainHookRepo) (bool, error) { +func (gruem *gerritChangeMergedEventMatcher) Match(hookRepo *commonmodels.MainHookRepo) (bool, error) { event := gruem.Event if event == nil { return false, fmt.Errorf("event doesn't match") @@ -180,6 +179,7 @@ func (gruem *gerritChangeMergedEventMatcher) Match(hookRepo commonmodels.MainHoo existEventNames = append(existEventNames, string(eventName)) } if sets.NewString(existEventNames...).Has(event.Type) { + hookRepo.Committer = event.Submitter.Username return true, nil } } @@ -187,7 +187,7 @@ func (gruem *gerritChangeMergedEventMatcher) Match(hookRepo commonmodels.MainHoo } func (gruem *gerritChangeMergedEventMatcher) UpdateTaskArgs( - product *commonmodels.Product, args *commonmodels.WorkflowTaskArgs, hookRepo commonmodels.MainHookRepo, requestID string, + product *commonmodels.Product, args *commonmodels.WorkflowTaskArgs, hookRepo *commonmodels.MainHookRepo, requestID string, ) *commonmodels.WorkflowTaskArgs { factory := &workflowArgsFactory{ workflow: gruem.Workflow, @@ -195,10 +195,11 @@ func (gruem *gerritChangeMergedEventMatcher) UpdateTaskArgs( } factory.Update(product, args, &types.Repository{ - CodehostID: hookRepo.CodehostID, - RepoName: hookRepo.RepoName, - RepoOwner: hookRepo.RepoOwner, - Branch: hookRepo.Branch, + CodehostID: hookRepo.CodehostID, + RepoName: hookRepo.RepoName, + RepoOwner: hookRepo.RepoOwner, + RepoNamespace: hookRepo.GetRepoNamespace(), + Branch: hookRepo.Branch, }) return args @@ -211,7 +212,7 @@ type gerritPatchsetCreatedEventMatcher struct { Event *patchsetCreatedEvent } -func (gpcem *gerritPatchsetCreatedEventMatcher) Match(commonmodels.MainHookRepo) (bool, error) { +func (gpcem *gerritPatchsetCreatedEventMatcher) Match(hookRepo *commonmodels.MainHookRepo) (bool, error) { event := gpcem.Event if event == nil { return false, fmt.Errorf("event doesn't match") @@ -223,24 +224,26 @@ func (gpcem *gerritPatchsetCreatedEventMatcher) Match(commonmodels.MainHookRepo) existEventNames = append(existEventNames, string(eventName)) } if sets.NewString(existEventNames...).Has(event.Type) { + hookRepo.Committer = event.Uploader.Username return true, nil } } return false, nil } -func (gpcem *gerritPatchsetCreatedEventMatcher) UpdateTaskArgs(product *commonmodels.Product, args *commonmodels.WorkflowTaskArgs, hookRepo commonmodels.MainHookRepo, requestID string) *commonmodels.WorkflowTaskArgs { +func (gpcem *gerritPatchsetCreatedEventMatcher) UpdateTaskArgs(product *commonmodels.Product, args *commonmodels.WorkflowTaskArgs, hookRepo *commonmodels.MainHookRepo, requestID string) *commonmodels.WorkflowTaskArgs { factory := &workflowArgsFactory{ workflow: gpcem.Workflow, reqID: requestID, } factory.Update(product, args, &types.Repository{ - CodehostID: hookRepo.CodehostID, - RepoName: hookRepo.RepoName, - RepoOwner: hookRepo.RepoOwner, - Branch: hookRepo.Branch, - PR: gpcem.Event.Change.Number, + CodehostID: hookRepo.CodehostID, + RepoName: hookRepo.RepoName, + RepoOwner: hookRepo.RepoOwner, + RepoNamespace: hookRepo.GetRepoNamespace(), + Branch: hookRepo.Branch, + PR: gpcem.Event.Change.Number, }) return args @@ -306,12 +309,12 @@ func TriggerWorkflowByGerritEvent(event *gerritTypeEvent, body []byte, uri, base if item.WorkflowArgs == nil { continue } - detail, err := codehost.GetCodehostDetail(item.MainRepo.CodehostID) + detail, err := systemconfig.New().GetCodeHost(item.MainRepo.CodehostID) if err != nil { log.Errorf("TriggerWorkflowByGerritEvent GetCodehostDetail err:%v", err) return err } - if detail.Source == gerrit.CodehostTypeGerrit { + if detail.Type == gerrit.CodehostTypeGerrit { log.Debugf("TriggerWorkflowByGerritEvent find gerrit hook in workflow %s", workflow.Name) matcher := createGerritEventMatcher(event, body, item, workflow, log) if matcher == nil { @@ -366,7 +369,7 @@ func TriggerWorkflowByGerritEvent(event *gerritTypeEvent, body []byte, uri, base mainRepo.RepoOwner = "" mainRepo.Revision = m.Event.PatchSet.Revision notification, _ = scmnotify.NewService().SendInitWebhookComment( - &mainRepo, m.Event.Change.Number, baseURI, false, false, log, + mainRepo, m.Event.Change.Number, baseURI, false, false, false, log, ) } } @@ -380,9 +383,10 @@ func TriggerWorkflowByGerritEvent(event *gerritTypeEvent, body []byte, uri, base workflowArgs.Source = setting.SourceFromGerrit workflowArgs.CodehostID = item.MainRepo.CodehostID workflowArgs.RepoOwner = item.MainRepo.RepoOwner + workflowArgs.RepoNamespace = item.MainRepo.GetRepoNamespace() workflowArgs.RepoName = item.MainRepo.RepoName - - if resp, err := workflowservice.CreateWorkflowTask(workflowArgs, setting.WebhookTaskCreator, permission.AnonymousUserID, false, log); err != nil { + workflowArgs.Committer = item.MainRepo.Committer + if resp, err := workflowservice.CreateWorkflowTask(workflowArgs, setting.WebhookTaskCreator, log); err != nil { log.Errorf("TriggerWorkflowByGerritEvent failed to create workflow task when receive push event %v due to %v ", event, err) errorList = multierror.Append(errorList, err) } else { @@ -411,7 +415,7 @@ func addWebHookUser(match gerritEventMatcher, domain string) { } } -func checkLatestTaskStaus(pipelineName, mergeRequestID, commitID string, detail *codehost.Detail, log *zap.SugaredLogger) bool { +func checkLatestTaskStaus(pipelineName, mergeRequestID, commitID string, detail *systemconfig.CodeHost, log *zap.SugaredLogger) bool { opt := &commonrepo.ListTaskOption{ PipelineName: pipelineName, Type: config.WorkflowType, @@ -435,7 +439,7 @@ func checkLatestTaskStaus(pipelineName, mergeRequestID, commitID string, detail } // 比较本次patchset 和 上一个触发任务的patchset 的change file是否相同 - cli := gerrit.NewClient(detail.Address, detail.OauthToken) + cli := gerrit.NewClient(detail.Address, detail.AccessToken, config.ProxyHTTPSAddr(), detail.EnableProxy) isDiff, err := cli.CompareTwoPatchset(mergeRequestID, commitID, tasks[0].TriggerBy.CommitID) if err != nil { log.Errorf("CompareTwoPatchset failed, mergeRequestID:%s, patchsetID:%s, oldPatchsetID:%s, err:%v", mergeRequestID, commitID, tasks[0].TriggerBy.CommitID, err) diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/gitee.go b/pkg/microservice/aslan/core/workflow/service/webhook/gitee.go new file mode 100644 index 0000000000000000000000000000000000000000..fa6713b91989960328829dd3d0eff5deac15500c --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/service/webhook/gitee.go @@ -0,0 +1,345 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "errors" + "fmt" + "io/ioutil" + "net/http" + "os" + "path" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/hashicorp/go-multierror" + "github.com/otiai10/copy" + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/config" + microserviceConfig "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/command" + gitservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/git" + environmentservice "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/service/service" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/gitee" +) + +func ProcessGiteeHook(payload []byte, req *http.Request, requestID string, log *zap.SugaredLogger) error { + token := req.Header.Get("X-Gitee-Token") + secret := gitservice.GetHookSecret() + + if secret != "" && token != secret { + return errors.New("token is illegal") + } + + eventType := gitee.HookEventType(req) + event, err := gitee.ParseHook(eventType, payload) + if err != nil { + return err + } + + baseURI := config.SystemAddress() + var errorList = &multierror.Error{} + var wg sync.WaitGroup + + switch event := event.(type) { + case *gitee.PushEvent: + //add webhook user + if len(event.Commits) > 0 { + webhookUser := &commonmodels.WebHookUser{ + Domain: req.Header.Get("X-Forwarded-Host"), + UserName: event.Commits[0].Author.Name, + Email: event.Commits[0].Author.Email, + Source: setting.SourceFromGitee, + CreatedAt: time.Now().Unix(), + } + commonrepo.NewWebHookUserColl().Upsert(webhookUser) + } + + if err := updateServiceTemplateByGiteeEvent(req.RequestURI, log); err != nil { + errorList = multierror.Append(errorList, err) + } + // build webhook + wg.Add(1) + go func() { + defer wg.Done() + if err = TriggerWorkflowByGiteeEvent(event, baseURI, requestID, log); err != nil { + errorList = multierror.Append(errorList, err) + } + }() + + //test webhook + wg.Add(1) + go func() { + defer wg.Done() + if err = TriggerTestByGiteeEvent(event, baseURI, requestID, log); err != nil { + errorList = multierror.Append(errorList, err) + } + }() + case *gitee.PullRequestEvent: + if event.Action != "open" && event.Action != "update" { + return fmt.Errorf("action %s is skipped", event.Action) + } + + if event.Action == "update" && event.ActionDesc == "target_branch_changed" { + return fmt.Errorf("action %s is skipped", event.Action) + } + + // build webhook + wg.Add(1) + go func() { + defer wg.Done() + if err = TriggerWorkflowByGiteeEvent(event, baseURI, requestID, log); err != nil { + errorList = multierror.Append(errorList, err) + } + }() + + //test webhook + wg.Add(1) + go func() { + defer wg.Done() + if err = TriggerTestByGiteeEvent(event, baseURI, requestID, log); err != nil { + errorList = multierror.Append(errorList, err) + } + }() + case *gitee.TagPushEvent: + // build webhook + wg.Add(1) + go func() { + defer wg.Done() + if err = TriggerWorkflowByGiteeEvent(event, baseURI, requestID, log); err != nil { + errorList = multierror.Append(errorList, err) + } + }() + + //test webhook + wg.Add(1) + go func() { + defer wg.Done() + if err = TriggerTestByGiteeEvent(event, baseURI, requestID, log); err != nil { + errorList = multierror.Append(errorList, err) + } + }() + } + wg.Wait() + return errorList.ErrorOrNil() +} + +func updateServiceTemplateByGiteeEvent(uri string, log *zap.SugaredLogger) error { + serviceTmpls, err := GetGiteeServiceTemplates() + if err != nil { + log.Errorf("failed to get gitee service templates, err: %s", err) + return err + } + errs := &multierror.Error{} + for _, serviceTmpl := range serviceTmpls { + service, err := commonservice.GetServiceTemplate(serviceTmpl.ServiceName, serviceTmpl.Type, serviceTmpl.ProductName, setting.ProductStatusDeleting, serviceTmpl.Revision, log) + if err != nil { + log.Errorf("failed to get gitee service templates, err:%s", err) + errs = multierror.Append(errs, err) + } + detail, err := systemconfig.New().GetCodeHost(service.CodehostID) + if err != nil { + log.Errorf("failed to get codehost detail err:%s", err) + errs = multierror.Append(errs, err) + } + newRepoName := fmt.Sprintf("%s-new", service.RepoName) + err = command.RunGitCmds(detail, service.RepoOwner, service.GetRepoNamespace(), newRepoName, service.BranchName, "origin") + if err != nil { + log.Errorf("failed to run git cmds err:%s", err) + errs = multierror.Append(errs, err) + } + + newBase, err := GetWorkspaceBasePath(newRepoName) + if err != nil { + errs = multierror.Append(errs, err) + } + + oldBase, err := GetWorkspaceBasePath(service.RepoName) + if err != nil { + errs = multierror.Append(errs, err) + err = command.RunGitCmds(detail, service.RepoOwner, service.GetRepoNamespace(), service.RepoName, service.BranchName, "origin") + if err != nil { + errs = multierror.Append(errs, err) + } + } + + filePath, err := os.Stat(path.Join(newBase, service.LoadPath)) + if err != nil { + errs = multierror.Append(errs, err) + } + + var newYamlContent string + var oldYamlContent string + if filePath.IsDir() { + if newFileContents, err := readAllFileContentUnderDir(path.Join(newBase, service.LoadPath)); err == nil { + newYamlContent = strings.Join(newFileContents, setting.YamlFileSeperator) + } else { + errs = multierror.Append(errs, err) + } + + if oldFileContents, err := readAllFileContentUnderDir(path.Join(oldBase, service.LoadPath)); err == nil { + oldYamlContent = strings.Join(oldFileContents, setting.YamlFileSeperator) + } else { + errs = multierror.Append(errs, err) + } + } else { + if contentBytes, err := ioutil.ReadFile(path.Join(newBase, service.LoadPath)); err == nil { + newYamlContent = string(contentBytes) + } else { + errs = multierror.Append(errs, err) + } + + if contentBytes, err := ioutil.ReadFile(path.Join(oldBase, service.LoadPath)); err == nil { + oldYamlContent = string(contentBytes) + } else { + errs = multierror.Append(errs, err) + } + } + + if strings.Compare(newYamlContent, oldYamlContent) != 0 { + log.Infof("Started to sync service template %s from gitee %s", service.ServiceName, service.LoadPath) + service.CreateBy = "system" + service.Yaml = newYamlContent + if err := SyncServiceTemplateFromGitee(service, log); err != nil { + errs = multierror.Append(errs, err) + } + } else { + log.Infof("Service template %s from gitee %s is not affected, no sync", service.ServiceName, service.LoadPath) + } + } + + return errs.ErrorOrNil() +} + +// GetGiteeServiceTemplates Get all service templates maintained in gitee +func GetGiteeServiceTemplates() ([]*commonmodels.Service, error) { + opt := &commonrepo.ServiceListOption{ + Source: setting.SourceFromGitee, + } + return commonrepo.NewServiceColl().ListMaxRevisions(opt) +} + +func GetWorkspaceBasePath(repoName string) (string, error) { + if strings.Contains(repoName, "/") { + repoName = strings.Replace(repoName, "/", "-", -1) + } + base := path.Join(microserviceConfig.S3StoragePath(), repoName) + + if _, err := os.Stat(base); os.IsNotExist(err) { + return base, err + } + + return base, nil +} + +func SyncServiceTemplateFromGitee(service *commonmodels.Service, log *zap.SugaredLogger) error { + if service.Source != setting.SourceFromGitee { + return fmt.Errorf("SyncServiceTemplateFromGitee service template is not from gitee,source:%s", service.Source) + } + + if _, err := syncGiteeLatestCommit(service); err != nil { + log.Errorf("SyncServiceTemplateFromGitee sync change log from gitee failed service %s, error: %s", service.ServiceName, err) + return err + } + + if service.Type == setting.K8SDeployType { + if err := ensureServiceTmpl(setting.WebhookTaskCreator, service, log); err != nil { + log.Errorf("SyncServiceTemplateFromGitee ensureServiceTmpl error: %s", err) + return e.ErrValidateTemplate.AddDesc(err.Error()) + } + + if err := commonrepo.NewServiceColl().Create(service); err != nil { + log.Errorf("SyncServiceTemplateFromGitee Failed to sync service %s from gitee path %s error: %s", service.ServiceName, service.SrcPath, err) + return e.ErrCreateTemplate.AddDesc(err.Error()) + } + + return environmentservice.AutoDeployYamlServiceToEnvs(service.CreateBy, "", service, log) + } + // remove old repo dir + oldRepoDir := filepath.Join(microserviceConfig.S3StoragePath(), service.RepoName) + os.RemoveAll(oldRepoDir) + // copy new repo data to old dir data + if err := os.MkdirAll(oldRepoDir, 0775); err != nil { + return err + } + + newRepoDir := filepath.Join(microserviceConfig.S3StoragePath(), service.RepoName+"-new") + if copyErr := copy.Copy(newRepoDir, oldRepoDir); copyErr != nil { + return copyErr + } + + if err := reloadServiceTmplFromGitee(service, log); err != nil { + return err + } + log.Infof("End of sync service template %s from gitee path %s", service.ServiceName, service.SrcPath) + return nil +} + +func syncGiteeLatestCommit(service *commonmodels.Service) (*systemconfig.CodeHost, error) { + if service.CodehostID == 0 { + return nil, fmt.Errorf("codehostId cannot be empty") + } + if service.RepoName == "" { + return nil, fmt.Errorf("repoName cannot be empty") + } + if service.BranchName == "" { + return nil, fmt.Errorf("branchName cannot be empty") + } + detail, err := systemconfig.New().GetCodeHost(service.CodehostID) + if err != nil { + return nil, err + } + + giteeCli := gitee.NewClient(detail.ID, detail.AccessToken, microserviceConfig.ProxyHTTPSAddr(), detail.EnableProxy) + commit, err := giteeCli.GetSingleBranch(detail.AccessToken, service.RepoOwner, service.RepoName, service.BranchName) + if err != nil { + return detail, err + } + + service.Commit = &commonmodels.Commit{ + SHA: commit.Commit.Sha, + Message: commit.Commit.Commit.Message, + } + return detail, nil +} + +func reloadServiceTmplFromGitee(svc *commonmodels.Service, log *zap.SugaredLogger) error { + _, err := service.CreateOrUpdateHelmServiceFromRepo(svc.ProductName, &service.HelmServiceCreationArgs{ + HelmLoadSource: service.HelmLoadSource{ + Source: setting.SourceFromGitee, + }, + CreatedBy: svc.CreateBy, + CreateFrom: &service.CreateFromRepo{ + CodehostID: svc.CodehostID, + Owner: svc.RepoOwner, + Repo: svc.RepoName, + Branch: svc.BranchName, + Paths: []string{svc.LoadPath}, + }, + }, true, log) + return err +} diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/gitee_testing_task.go b/pkg/microservice/aslan/core/workflow/service/webhook/gitee_testing_task.go new file mode 100644 index 0000000000000000000000000000000000000000..11d275686a593665d4382a4fe57c678e17defbba --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/service/webhook/gitee_testing_task.go @@ -0,0 +1,291 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "regexp" + "strconv" + + "github.com/hashicorp/go-multierror" + "github.com/koderover/zadig/pkg/setting" + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/scmnotify" + testingservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/testing/service" + "github.com/koderover/zadig/pkg/tool/gitee" +) + +type giteePushEventMatcherForTesting struct { + log *zap.SugaredLogger + testing *commonmodels.Testing + event *gitee.PushEvent +} + +func (gpem *giteePushEventMatcherForTesting) Match(hookRepo *commonmodels.MainHookRepo) (bool, error) { + ev := gpem.event + if (hookRepo.RepoOwner + "/" + hookRepo.RepoName) == ev.Repository.FullName { + if !EventConfigured(hookRepo, config.HookEventPush) { + return false, nil + } + isRegular := hookRepo.IsRegular + if !isRegular && hookRepo.Branch != getBranchFromRef(ev.Ref) { + return false, nil + } + + if isRegular { + matched, err := regexp.MatchString(hookRepo.Branch, getBranchFromRef(ev.Ref)) + if err != nil || !matched { + return false, nil + } + } + hookRepo.Branch = getBranchFromRef(ev.Ref) + var changedFiles []string + for _, commit := range ev.Commits { + changedFiles = append(changedFiles, commit.Added...) + changedFiles = append(changedFiles, commit.Removed...) + changedFiles = append(changedFiles, commit.Modified...) + } + + return MatchChanges(hookRepo, changedFiles), nil + } + + return false, nil +} + +func (gpem *giteePushEventMatcherForTesting) UpdateTaskArgs(args *commonmodels.TestTaskArgs, requestID string) *commonmodels.TestTaskArgs { + factory := &testArgsFactory{ + testing: gpem.testing, + reqID: requestID, + } + + factory.Update(args) + return args +} + +type giteeTagEventMatcherForTesting struct { + log *zap.SugaredLogger + testing *commonmodels.Testing + event *gitee.TagPushEvent +} + +func (gtem giteeTagEventMatcherForTesting) Match(hookRepo *commonmodels.MainHookRepo) (bool, error) { + ev := gtem.event + if (hookRepo.RepoOwner + "/" + hookRepo.RepoName) == ev.Project.PathWithNamespace { + if !EventConfigured(hookRepo, config.HookEventTag) { + return false, nil + } + isRegular := hookRepo.IsRegular + if !isRegular && hookRepo.Branch != ev.Project.DefaultBranch { + return false, nil + } + + if isRegular { + matched, err := regexp.MatchString(hookRepo.Branch, ev.Project.DefaultBranch) + if err != nil || !matched { + return false, nil + } + } + hookRepo.Branch = ev.Project.DefaultBranch + return true, nil + } + + return false, nil +} + +func (gtem giteeTagEventMatcherForTesting) UpdateTaskArgs(args *commonmodels.TestTaskArgs, requestID string) *commonmodels.TestTaskArgs { + factory := &testArgsFactory{ + testing: gtem.testing, + reqID: requestID, + } + + factory.Update(args) + return args +} + +type giteeMergeEventMatcherForTesting struct { + diffFunc giteePullRequestDiffFunc + log *zap.SugaredLogger + testing *commonmodels.Testing + event *gitee.PullRequestEvent +} + +func (gmem *giteeMergeEventMatcherForTesting) Match(hookRepo *commonmodels.MainHookRepo) (bool, error) { + ev := gmem.event + // TODO: match codehost + if (hookRepo.RepoOwner + "/" + hookRepo.RepoName) == ev.PullRequest.Base.Repo.FullName { + if !EventConfigured(hookRepo, config.HookEventPr) { + return false, nil + } + + isRegular := hookRepo.IsRegular + if !isRegular && hookRepo.Branch != ev.PullRequest.Base.Ref { + return false, nil + } + + if isRegular { + matched, err := regexp.MatchString(hookRepo.Branch, ev.PullRequest.Base.Ref) + if err != nil || !matched { + return false, nil + } + } + hookRepo.Branch = ev.PullRequest.Base.Ref + if ev.PullRequest.State == "open" { + var changedFiles []string + changedFiles, err := gmem.diffFunc(ev, hookRepo.CodehostID) + if err != nil { + gmem.log.Warnf("failed to get changes of event %v", ev) + return false, err + } + gmem.log.Debugf("succeed to get %d changes in merge event", len(changedFiles)) + + return MatchChanges(hookRepo, changedFiles), nil + } + + } + return false, nil +} + +func (gmem *giteeMergeEventMatcherForTesting) UpdateTaskArgs(args *commonmodels.TestTaskArgs, requestID string) *commonmodels.TestTaskArgs { + factory := &testArgsFactory{ + testing: gmem.testing, + reqID: requestID, + } + + args = factory.Update(args) + + return args +} + +func createGiteeEventMatcherForTesting(event interface{}, diffSrv giteePullRequestDiffFunc, testing *commonmodels.Testing, log *zap.SugaredLogger) gitEventMatcherForTesting { + switch evt := event.(type) { + case *gitee.PushEvent: + return &giteePushEventMatcherForTesting{ + testing: testing, + log: log, + event: evt, + } + case *gitee.PullRequestEvent: + return &giteeMergeEventMatcherForTesting{ + diffFunc: diffSrv, + log: log, + event: evt, + testing: testing, + } + case *gitee.TagPushEvent: + return &giteeTagEventMatcherForTesting{ + testing: testing, + log: log, + event: evt, + } + } + + return nil +} + +func TriggerTestByGiteeEvent(event interface{}, baseURI, requestID string, log *zap.SugaredLogger) error { + // 1. find configured testing + testingList, err := commonrepo.NewTestingColl().List(&commonrepo.ListTestOption{}) + if err != nil { + log.Errorf("failed to list testing,err: %s", err) + return err + } + + mErr := &multierror.Error{} + diffSrv := func(PullRequestEvent *gitee.PullRequestEvent, codehostId int) ([]string, error) { + return findChangedFilesOfPullRequestEvent(PullRequestEvent, codehostId) + } + + var notification *commonmodels.Notification + for _, testing := range testingList { + if testing.HookCtl != nil && testing.HookCtl.Enabled { + log.Infof("find %d hooks in testing %s", len(testing.HookCtl.Items), testing.Name) + for _, item := range testing.HookCtl.Items { + if item.TestArgs == nil { + continue + } + + // 2. match webhook + matcher := createGiteeEventMatcherForTesting(event, diffSrv, testing, log) + if matcher == nil { + continue + } + + if matches, err := matcher.Match(item.MainRepo); err != nil { + mErr = multierror.Append(mErr, err) + } else if matches { + log.Infof("event match hook %v of %s", item.MainRepo, testing.Name) + var mergeRequestID, commitID string + if ev, isPr := event.(*gitee.PullRequestEvent); isPr { + // If it is a merge request, and the webhook trigger is configured with automatic cancellation, + // It is necessary to confirm whether the task triggered by the commit of the merge request before this commit has been processed, and it will be cancelled if it is not processed. + mergeRequestID = strconv.Itoa(ev.PullRequest.Number) + commitID = ev.PullRequest.Head.Sha + autoCancelOpt := &AutoCancelOpt{ + MergeRequestID: mergeRequestID, + CommitID: commitID, + TaskType: config.TestType, + MainRepo: item.MainRepo, + TestArgs: item.TestArgs, + } + err := AutoCancelTask(autoCancelOpt, log) + if err != nil { + log.Errorf("failed to auto cancel testing task when receive event %v due to %s ", event, err) + mErr = multierror.Append(mErr, err) + } + + if notification == nil { + notification, err = scmnotify.NewService().SendInitWebhookComment( + item.MainRepo, ev.PullRequest.Number, baseURI, false, true, false, log, + ) + if err != nil { + log.Errorf("failed to init webhook comment due to %s", err) + mErr = multierror.Append(mErr, err) + } + } + } + + if notification != nil { + item.TestArgs.NotificationID = notification.ID.Hex() + } + + args := matcher.UpdateTaskArgs(item.TestArgs, requestID) + args.MergeRequestID = mergeRequestID + args.CommitID = commitID + args.Source = setting.SourceFromGitee + args.CodehostID = item.MainRepo.CodehostID + args.RepoOwner = item.MainRepo.RepoOwner + args.RepoName = item.MainRepo.RepoName + + // 3. create task with args + if resp, err := testingservice.CreateTestTask(args, log); err != nil { + log.Errorf("failed to create testing task when receive event %v due to %s ", event, err) + mErr = multierror.Append(mErr, err) + } else { + log.Infof("succeed to create task %v", resp) + } + } else { + log.Debugf("event not matches %v", item.MainRepo) + } + } + } + } + + return mErr.ErrorOrNil() +} diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/gitee_workflow_task.go b/pkg/microservice/aslan/core/workflow/service/webhook/gitee_workflow_task.go new file mode 100644 index 0000000000000000000000000000000000000000..ca073b737df70a1ad472b03f6599d1aab72da8d8 --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/service/webhook/gitee_workflow_task.go @@ -0,0 +1,360 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "fmt" + "regexp" + "strconv" + "strings" + + "github.com/hashicorp/go-multierror" + "github.com/koderover/zadig/pkg/types" + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/scmnotify" + workflowservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" + "github.com/koderover/zadig/pkg/tool/gitee" +) + +type giteePullRequestDiffFunc func(event *gitee.PullRequestEvent, id int) ([]string, error) + +type giteePushEventMatcher struct { + log *zap.SugaredLogger + workflow *commonmodels.Workflow + event *gitee.PushEvent +} + +func (gpem *giteePushEventMatcher) Match(hookRepo *commonmodels.MainHookRepo) (bool, error) { + ev := gpem.event + if (hookRepo.RepoOwner + "/" + hookRepo.RepoName) == ev.Repository.FullName { + if !EventConfigured(hookRepo, config.HookEventPush) { + return false, nil + } + + isRegular := hookRepo.IsRegular + if !isRegular && hookRepo.Branch != getBranchFromRef(ev.Ref) { + return false, nil + } + if isRegular { + // Do not use regexp.MustCompile to avoid panic + matched, err := regexp.MatchString(hookRepo.Branch, getBranchFromRef(ev.Ref)) + if err != nil || !matched { + return false, nil + } + } + hookRepo.Branch = getBranchFromRef(ev.Ref) + hookRepo.Committer = ev.Pusher.Name + var changedFiles []string + for _, commit := range ev.Commits { + changedFiles = append(changedFiles, commit.Added...) + changedFiles = append(changedFiles, commit.Removed...) + changedFiles = append(changedFiles, commit.Modified...) + } + return MatchChanges(hookRepo, changedFiles), nil + } + + return false, nil +} + +func (gpem *giteePushEventMatcher) UpdateTaskArgs( + product *commonmodels.Product, args *commonmodels.WorkflowTaskArgs, hookRepo *commonmodels.MainHookRepo, requestID string, +) *commonmodels.WorkflowTaskArgs { + factory := &workflowArgsFactory{ + workflow: gpem.workflow, + reqID: requestID, + } + + factory.Update(product, args, &types.Repository{ + CodehostID: hookRepo.CodehostID, + RepoName: hookRepo.RepoName, + RepoOwner: hookRepo.RepoOwner, + Branch: hookRepo.Branch, + }) + + return args +} + +type giteeMergeEventMatcher struct { + diffFunc giteePullRequestDiffFunc + log *zap.SugaredLogger + workflow *commonmodels.Workflow + event *gitee.PullRequestEvent +} + +func (gmem *giteeMergeEventMatcher) Match(hookRepo *commonmodels.MainHookRepo) (bool, error) { + ev := gmem.event + if (hookRepo.RepoOwner + "/" + hookRepo.RepoName) == ev.PullRequest.Base.Repo.FullName { + if !EventConfigured(hookRepo, config.HookEventPr) { + return false, nil + } + + isRegular := hookRepo.IsRegular + if !isRegular && hookRepo.Branch != ev.PullRequest.Base.Ref { + return false, nil + } + if isRegular { + matched, err := regexp.MatchString(hookRepo.Branch, ev.PullRequest.Base.Ref) + if err != nil || !matched { + return false, nil + } + } + hookRepo.Branch = ev.PullRequest.Base.Ref + hookRepo.Committer = ev.PullRequest.User.Login + if ev.PullRequest.State == "open" { + var changedFiles []string + changedFiles, err := gmem.diffFunc(ev, hookRepo.CodehostID) + if err != nil { + gmem.log.Warnf("failed to get changes of event %v", ev) + return false, err + } + gmem.log.Debugf("succeed to get %d changes in merge event", len(changedFiles)) + + return MatchChanges(hookRepo, changedFiles), nil + } + } + return false, nil +} + +func (gmem *giteeMergeEventMatcher) UpdateTaskArgs( + product *commonmodels.Product, args *commonmodels.WorkflowTaskArgs, hookRepo *commonmodels.MainHookRepo, requestID string, +) *commonmodels.WorkflowTaskArgs { + factory := &workflowArgsFactory{ + workflow: gmem.workflow, + reqID: requestID, + } + + args = factory.Update(product, args, &types.Repository{ + CodehostID: hookRepo.CodehostID, + RepoName: hookRepo.RepoName, + RepoOwner: hookRepo.RepoOwner, + Branch: hookRepo.Branch, + PR: gmem.event.PullRequest.Number, + }) + + return args +} + +type giteeTagEventMatcher struct { + log *zap.SugaredLogger + workflow *commonmodels.Workflow + event *gitee.TagPushEvent +} + +func (gtem giteeTagEventMatcher) Match(hookRepo *commonmodels.MainHookRepo) (bool, error) { + ev := gtem.event + if (hookRepo.RepoOwner + "/" + hookRepo.RepoName) == ev.Repository.FullName { + if !EventConfigured(hookRepo, config.HookEventTag) { + return false, nil + } + + isRegular := hookRepo.IsRegular + if !isRegular && hookRepo.Branch != ev.Repository.DefaultBranch { + return false, nil + } + if isRegular { + // Do not use regexp.MustCompile to avoid panic + matched, err := regexp.MatchString(hookRepo.Branch, ev.Repository.DefaultBranch) + if err != nil || !matched { + return false, nil + } + } + hookRepo.Tag = getTagFromRef(ev.Ref) + hookRepo.Committer = ev.Sender.Name + + return true, nil + } + + return false, nil +} + +func (gtem giteeTagEventMatcher) UpdateTaskArgs(product *commonmodels.Product, args *commonmodels.WorkflowTaskArgs, hookRepo *commonmodels.MainHookRepo, requestID string) *commonmodels.WorkflowTaskArgs { + factory := &workflowArgsFactory{ + workflow: gtem.workflow, + reqID: requestID, + } + + factory.Update(product, args, &types.Repository{ + CodehostID: hookRepo.CodehostID, + RepoName: hookRepo.RepoName, + RepoOwner: hookRepo.RepoOwner, + Branch: hookRepo.Branch, + Tag: hookRepo.Tag, + }) + + return args +} + +func createGiteeEventMatcher( + event interface{}, diffSrv giteePullRequestDiffFunc, workflow *commonmodels.Workflow, log *zap.SugaredLogger, +) gitEventMatcher { + switch evt := event.(type) { + case *gitee.PushEvent: + return &giteePushEventMatcher{ + workflow: workflow, + log: log, + event: evt, + } + case *gitee.PullRequestEvent: + return &giteeMergeEventMatcher{ + diffFunc: diffSrv, + log: log, + event: evt, + workflow: workflow, + } + case *gitee.TagPushEvent: + return &giteeTagEventMatcher{ + workflow: workflow, + log: log, + event: evt, + } + } + + return nil +} + +func TriggerWorkflowByGiteeEvent(event interface{}, baseURI, requestID string, log *zap.SugaredLogger) error { + workflowList, err := commonrepo.NewWorkflowColl().List(&commonrepo.ListWorkflowOption{}) + if err != nil { + log.Errorf("failed to list workflow %v", err) + return err + } + + mErr := &multierror.Error{} + diffSrv := func(pullRequestEvent *gitee.PullRequestEvent, codehostId int) ([]string, error) { + return findChangedFilesOfPullRequestEvent(pullRequestEvent, codehostId) + } + + var notification *commonmodels.Notification + for _, workflow := range workflowList { + if workflow.HookCtl != nil && workflow.HookCtl.Enabled { + log.Debugf("find %d hooks in workflow %s", len(workflow.HookCtl.Items), workflow.Name) + for _, item := range workflow.HookCtl.Items { + if item.WorkflowArgs == nil { + continue + } + + matcher := createGiteeEventMatcher(event, diffSrv, workflow, log) + if matcher == nil { + continue + } + + if matches, err := matcher.Match(item.MainRepo); err != nil { + mErr = multierror.Append(mErr, err) + } else if matches { + log.Infof("event match hook %v of %s", item.MainRepo, workflow.Name) + namespace := strings.Split(item.WorkflowArgs.Namespace, ",")[0] + opt := &commonrepo.ProductFindOptions{Name: workflow.ProductTmplName, EnvName: namespace} + var prod *commonmodels.Product + if prod, err = commonrepo.NewProductColl().Find(opt); err != nil { + log.Warnf("can't find environment %s-%s", item.WorkflowArgs.Namespace, workflow.ProductTmplName) + continue + } + + var mergeRequestID, commitID string + var hookPayload *commonmodels.HookPayload + if ev, isPr := event.(*gitee.PullRequestEvent); isPr { + if ev.PullRequest != nil && ev.PullRequest.Number != 0 && ev.PullRequest.Head != nil && ev.PullRequest.Head.Sha != "" { + mergeRequestID = strconv.Itoa(ev.PullRequest.Number) + commitID = ev.PullRequest.Head.Sha + autoCancelOpt := &AutoCancelOpt{ + MergeRequestID: mergeRequestID, + CommitID: commitID, + TaskType: config.WorkflowType, + MainRepo: item.MainRepo, + WorkflowArgs: item.WorkflowArgs, + } + err := AutoCancelTask(autoCancelOpt, log) + if err != nil { + log.Errorf("failed to auto cancel workflow task when receive event due to %v ", err) + mErr = multierror.Append(mErr, err) + } + + if notification == nil { + notification, err = scmnotify.NewService().SendInitWebhookComment( + item.MainRepo, ev.PullRequest.Number, baseURI, false, false, false, log, + ) + if err != nil { + log.Errorf("failed to init webhook comment due to %s", err) + mErr = multierror.Append(mErr, err) + } + } + } + + if notification != nil { + item.WorkflowArgs.NotificationID = notification.ID.Hex() + } + + hookPayload = &commonmodels.HookPayload{ + Owner: ev.Repository.Owner.Login, + Repo: ev.Repository.Name, + Branch: ev.PullRequest.Base.Ref, + Ref: ev.PullRequest.Head.Sha, + IsPr: true, + } + } + + args := matcher.UpdateTaskArgs(prod, item.WorkflowArgs, item.MainRepo, requestID) + args.MergeRequestID = mergeRequestID + args.CommitID = commitID + args.Source = setting.SourceFromGitee + args.CodehostID = item.MainRepo.CodehostID + args.RepoOwner = item.MainRepo.RepoOwner + args.RepoName = item.MainRepo.RepoName + args.Committer = item.MainRepo.Committer + args.HookPayload = hookPayload + + // 3. create task with args + if resp, err := workflowservice.CreateWorkflowTask(args, setting.WebhookTaskCreator, log); err != nil { + log.Errorf("failed to create workflow task when receive push event due to %v ", err) + mErr = multierror.Append(mErr, err) + } else { + log.Infof("succeed to create task %v", resp) + } + } else { + log.Debugf("event not matches %v", item.MainRepo) + } + } + } + } + + return mErr.ErrorOrNil() +} + +func findChangedFilesOfPullRequestEvent(event *gitee.PullRequestEvent, codehostID int) ([]string, error) { + detail, err := systemconfig.New().GetCodeHost(codehostID) + if err != nil { + return nil, fmt.Errorf("failed to find codehost %d: %v", codehostID, err) + } + + giteeCli := gitee.NewClient(detail.ID, detail.AccessToken, config.ProxyHTTPSAddr(), detail.EnableProxy) + commitComparison, err := giteeCli.GetReposOwnerRepoCompareBaseHead(detail.AccessToken, event.Project.Namespace, event.Project.Name, event.PullRequest.Base.Sha, event.PullRequest.Head.Sha) + if err != nil { + return nil, fmt.Errorf("failed to get changes from gitee, err: %v", err) + } + + changeFiles := make([]string, 0) + for _, commitFile := range commitComparison.Files { + changeFiles = append(changeFiles, commitFile.Filename) + } + return changeFiles, nil +} diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/github.go b/pkg/microservice/aslan/core/workflow/service/webhook/github.go index f963ff2e80a8d242210046dd70ceaad03622ac17..382136a3da7df41cb6713336e5d4ccc6c64de5e3 100644 --- a/pkg/microservice/aslan/core/workflow/service/webhook/github.go +++ b/pkg/microservice/aslan/core/workflow/service/webhook/github.go @@ -33,12 +33,10 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/collie" gitservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/git" workflowservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/codehost" - "github.com/koderover/zadig/pkg/shared/poetry" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" e "github.com/koderover/zadig/pkg/tool/errors" githubtool "github.com/koderover/zadig/pkg/tool/git/github" "github.com/koderover/zadig/pkg/types" @@ -50,32 +48,31 @@ const ( payloadFormParam = "payload" ) -func ProcessGithubHook(payload []byte, req *http.Request, requestID string, log *zap.SugaredLogger) (output string, err error) { +func ProcessGithubHook(payload []byte, req *http.Request, requestID string, log *zap.SugaredLogger) (string, error) { hookType := github.WebHookType(req) if hookType == "integration_installation" || hookType == "installation" || hookType == "ping" { - output = fmt.Sprintf("event %s received", hookType) - return + return fmt.Sprintf("event %s received", hookType), nil } hookSecret := gitservice.GetHookSecret() - if hookSecret == "" { var headers []string for header := range req.Header { headers = append(headers, fmt.Sprintf("%s: %s", header, req.Header.Get(header))) } - log.Infof("[Webhook] hook headers: \n %s", strings.Join(headers, "\n ")) } - err = validateSecret(payload, []byte(hookSecret), req) + err := validateSecret(payload, []byte(hookSecret), req) if err != nil { - return + log.Errorf("Failed to validate secret, err: %s", err) + return "", err } event, err := github.ParseWebHook(github.WebHookType(req), payload) if err != nil { - return + log.Errorf("Failed to parse webhook, err: %s", err) + return "", err } deliveryID := github.DeliveryID(req) @@ -86,15 +83,13 @@ func ProcessGithubHook(payload []byte, req *http.Request, requestID string, log switch et := event.(type) { case *github.PullRequestEvent: if *et.Action != "opened" && *et.Action != "synchronize" && *et.Action != "reopened" { - output = fmt.Sprintf("action %s is skipped", *et.Action) - return + return fmt.Sprintf("action %s is skipped", *et.Action), nil } tasks, err = prEventToPipelineTasks(et, requestID, log) if err != nil { log.Errorf("prEventToPipelineTasks error: %v", err) - err = e.ErrGithubWebHook.AddErr(err) - return + return "", e.ErrGithubWebHook.AddErr(err) } case *github.PushEvent: @@ -112,40 +107,34 @@ func ProcessGithubHook(payload []byte, req *http.Request, requestID string, log tasks, err = pushEventToPipelineTasks(et, requestID, log) if err != nil { - log.Infof("pushEventToPipelineTasks error: %v", err) - err = e.ErrGithubWebHook.AddErr(err) - return + log.Errorf("pushEventToPipelineTasks error: %v", err) + return "", e.ErrGithubWebHook.AddErr(err) } case *github.CheckRunEvent: // The action performed. Can be "created", "updated", "rerequested" or "requested_action". if *et.Action != "rerequested" { - output = fmt.Sprintf("action %s is skipped", *et.Action) - return + return fmt.Sprintf("action %s is skipped", *et.Action), nil } id := et.CheckRun.GetExternalID() items := strings.Split(id, "/") if len(items) != 2 { - err = fmt.Errorf("invalid CheckRun ExternalID %s", id) - return + return "", fmt.Errorf("invalid CheckRun ExternalID %s", id) } pipeName := items[0] var taskID int64 taskID, err = strconv.ParseInt(items[1], 10, 64) if err != nil { - err = fmt.Errorf("invalid taskID in CheckRun ExternalID %s", id) - return + return "", fmt.Errorf("invalid taskID in CheckRun ExternalID %s", id) } if err = workflowservice.RestartPipelineTaskV2("CheckRun", taskID, pipeName, config.SingleType, log); err != nil { - err = e.ErrGithubWebHook.AddErr(err) - return + return "", e.ErrGithubWebHook.AddErr(err) } default: - output = fmt.Sprintf("event %s not support", hookType) - return + return fmt.Sprintf("event %s not support", hookType), nil } for _, task := range tasks { @@ -159,7 +148,7 @@ func ProcessGithubHook(payload []byte, req *http.Request, requestID string, log log.Infof("[Webhook] %s triggered task %s:%d", deliveryID, task.PipelineName, resp.TaskID) } - return + return "", nil } func validateSecret(payload, secretKey []byte, r *http.Request) error { @@ -183,13 +172,13 @@ func prEventToPipelineTasks(event *github.PullRequestEvent, requestID string, lo commitMessage = *event.PullRequest.Title ) - address, err := util.GetAddress(event.Repo.GetURL()) + address, err := util.GetAddress(event.Repo.GetHTMLURL()) if err != nil { log.Errorf("GetAddress failed, url: %s, err: %s", event.Repo.GetURL(), err) return nil, err } - ch, err := codehost.GetCodeHostInfo( - &codehost.Option{CodeHostType: poetry.GitHubProvider, Address: address, Namespace: owner}) + ch, err := systemconfig.GetCodeHostInfo( + &systemconfig.Option{CodeHostType: systemconfig.GitHubProvider, Address: address, Namespace: owner}) if err != nil { log.Errorf("GetCodeHostInfo failed, err: %v", err) return nil, err @@ -210,6 +199,7 @@ func prEventToPipelineTasks(event *github.PullRequestEvent, requestID string, lo eventRepo := &types.Repository{ RepoOwner: owner, + RepoNamespace: owner, RepoName: repo, Branch: branch, PR: prNum, @@ -327,8 +317,8 @@ func pushEventToPipelineTasks(event *github.PushEvent, requestID string, log *za log.Errorf("GetAddress failed, url: %s, err: %s", event.Repo.GetURL(), err) return nil, err } - ch, err := codehost.GetCodeHostInfo( - &codehost.Option{CodeHostType: poetry.GitHubProvider, Address: address, Namespace: owner}) + ch, err := systemconfig.GetCodeHostInfo( + &systemconfig.Option{CodeHostType: systemconfig.GitHubProvider, Address: address, Namespace: owner}) if err != nil { log.Errorf("GetCodeHostInfo failed, err: %v", err) return nil, err @@ -342,6 +332,7 @@ func pushEventToPipelineTasks(event *github.PushEvent, requestID string, log *za eventRepo := &types.Repository{ RepoOwner: owner, + RepoNamespace: owner, RepoName: repo, Branch: branch, CommitID: commitID, @@ -391,6 +382,69 @@ func pushEventCommitsFiles(e *github.PushEvent) []string { return files } +func ProcessGithubWebHookForTest(payload []byte, req *http.Request, requestID string, log *zap.SugaredLogger) error { + hookType := github.WebHookType(req) + if hookType == "integration_installation" || hookType == "installation" || hookType == "ping" { + return nil + } + + err := validateSecret(payload, []byte(gitservice.GetHookSecret()), req) + if err != nil { + return err + } + + event, err := github.ParseWebHook(github.WebHookType(req), payload) + if err != nil { + return err + } + + switch et := event.(type) { + case *github.PullRequestEvent, *github.PushEvent, *github.CreateEvent: + if err = TriggerTestByGithubEvent(et, requestID, log); err != nil { + log.Errorf("TriggerTestByGithubEvent error: %s", err) + return e.ErrGithubWebHook.AddErr(err) + } + default: + log.Warn("Unsupported event type") + return nil + } + return nil +} + +func ProcessGithubWebhookForScanning(payload []byte, req *http.Request, requestID string, log *zap.SugaredLogger) error { + hookType := github.WebHookType(req) + if hookType == "integration_installation" || hookType == "installation" || hookType == "ping" { + return nil + } + + err := validateSecret(payload, []byte(gitservice.GetHookSecret()), req) + if err != nil { + return err + } + + event, err := github.ParseWebHook(github.WebHookType(req), payload) + if err != nil { + log.Errorf("Failed to parse webhook, err: %s", err) + return err + } + + deliveryID := github.DeliveryID(req) + + log.Infof("[Webhook] event: %s delivery id: %s received for scanning trigger", hookType, deliveryID) + + switch et := event.(type) { + case *github.PullRequestEvent, *github.PushEvent, *github.CreateEvent: + if err = TriggerScanningByGithubEvent(et, requestID, log); err != nil { + log.Errorf("TriggerScanningByGithubEvent error: %s", err) + return e.ErrGithubWebHook.AddErr(err) + } + default: + log.Warn("Unsupported event type") + return nil + } + return nil +} + func ProcessGithubWebHook(payload []byte, req *http.Request, requestID string, log *zap.SugaredLogger) error { forwardedProto := req.Header.Get("X-Forwarded-Proto") forwardedHost := req.Header.Get("X-Forwarded-Host") @@ -411,8 +465,6 @@ func ProcessGithubWebHook(payload []byte, req *http.Request, requestID string, l return err } - go collie.CallGithubWebHook(forwardedProto, forwardedHost, payload, github.WebHookType(req), log) - deliveryID := github.DeliveryID(req) log.Infof("[Webhook] event: %s delivery id: %s received", hookType, deliveryID) @@ -429,11 +481,9 @@ func ProcessGithubWebHook(payload []byte, req *http.Request, requestID string, l } case *github.PushEvent: - //触发更新服务模板webhook - if pushEvent, isPush := event.(*github.PushEvent); isPush { - if err = updateServiceTemplateByGithubPush(pushEvent, log); err != nil { - log.Errorf("updateServiceTemplateByGithubPush failed, error:%v", err) - } + // sync service template + if err = updateServiceTemplateByGithubPush(et, log); err != nil { + log.Errorf("updateServiceTemplateByGithubPush failed, error:%v", err) } //add webhook user @@ -452,6 +502,12 @@ func ProcessGithubWebHook(payload []byte, req *http.Request, requestID string, l log.Infof("pushEventToPipelineTasks error: %v", err) return e.ErrGithubWebHook.AddErr(err) } + case *github.CreateEvent: + err = TriggerWorkflowByGithubEvent(et, baseURI, deliveryID, requestID, log) + if err != nil { + log.Errorf("tagEventToPipelineTasks error: %s", err) + return e.ErrGithubWebHook.AddErr(err) + } } return nil } @@ -460,53 +516,12 @@ type AutoCancelOpt struct { MergeRequestID string CommitID string TaskType config.PipelineType - MainRepo commonmodels.MainHookRepo + MainRepo *commonmodels.MainHookRepo WorkflowArgs *commonmodels.WorkflowTaskArgs TestArgs *commonmodels.TestTaskArgs -} - -func getProductTargetMap(prod *commonmodels.Product) map[string][]commonmodels.DeployEnv { - resp := make(map[string][]commonmodels.DeployEnv) - if prod.Source == setting.SourceFromExternal { - services, _ := commonrepo.NewServiceColl().ListExternalWorkloadsBy(prod.ProductName, prod.EnvName) - for _, service := range services { - for _, container := range service.Containers { - env := service.ServiceName + "/" + container.Name - deployEnv := commonmodels.DeployEnv{Type: setting.K8SDeployType, Env: env} - target := strings.Join([]string{service.ProductName, service.ServiceName, container.Name}, SplitSymbol) - resp[target] = append(resp[target], deployEnv) - } - } - return resp - } - for _, services := range prod.Services { - for _, serviceObj := range services { - switch serviceObj.Type { - case setting.K8SDeployType: - for _, container := range serviceObj.Containers { - env := serviceObj.ServiceName + "/" + container.Name - deployEnv := commonmodels.DeployEnv{Type: setting.K8SDeployType, Env: env} - - target := fmt.Sprintf("%s%s%s%s%s", prod.ProductName, SplitSymbol, serviceObj.ServiceName, SplitSymbol, container.Name) - - resp[target] = append(resp[target], deployEnv) - } - case setting.PMDeployType: - deployEnv := commonmodels.DeployEnv{Type: setting.PMDeployType, Env: serviceObj.ServiceName} - target := fmt.Sprintf("%s%s%s%s%s", prod.ProductName, SplitSymbol, serviceObj.ServiceName, SplitSymbol, serviceObj.ServiceName) - resp[target] = append(resp[target], deployEnv) - case setting.HelmDeployType: - for _, container := range serviceObj.Containers { - env := serviceObj.ServiceName + "/" + container.Name - deployEnv := commonmodels.DeployEnv{Type: setting.HelmDeployType, Env: env} - - target := fmt.Sprintf("%s%s%s%s%s", prod.ProductName, SplitSymbol, serviceObj.ServiceName, SplitSymbol, container.Name) - resp[target] = append(resp[target], deployEnv) - } - } - } - } - return resp + IsYaml bool + AutoCancel bool + YamlHookPath string } func updateServiceTemplateByGithubPush(pushEvent *github.PushEvent, log *zap.SugaredLogger) error { @@ -535,15 +550,14 @@ func updateServiceTemplateByGithubPush(pushEvent *github.PushEvent, log *zap.Sug errs := &multierror.Error{} for _, service := range serviceTmpls { - srcPath := service.SrcPath - _, _, _, _, path, _, err := GetOwnerRepoBranchPath(srcPath) + path, err := getServiceSrcPath(service) if err != nil { errs = multierror.Append(errs, err) } // 判断PushEvent的Diffs中是否包含该服务模板的src_path affected := false for _, changeFile := range changeFiles { - if strings.Contains(changeFile, path) { + if subElem(path, changeFile) { affected = true break } @@ -566,13 +580,14 @@ func updateServiceTemplateByGithubPush(pushEvent *github.PushEvent, log *zap.Sug func GetGithubServiceTemplates() ([]*commonmodels.Service, error) { opt := &commonrepo.ServiceListOption{ - Type: setting.K8SDeployType, Source: setting.SourceFromGithub, } return commonrepo.NewServiceColl().ListMaxRevisions(opt) } // GetOwnerRepoBranchPath 获取gitlab路径中的owner、repo、branch和path +// return address, owner, repo, branch, path, pathType +// Note this function needs be optimized badly! func GetOwnerRepoBranchPath(URL string) (string, string, string, string, string, string, error) { if !strings.Contains(URL, "https") && !strings.Contains(URL, "http") { return "", "", "", "", "", "", fmt.Errorf("url is illegal") @@ -645,11 +660,7 @@ func SyncServiceTemplateFromGithub(service *commonmodels.Service, latestCommitID log.Errorf("ensure github serviceTmpl failed, error: %+v", err) return e.ErrValidateTemplate.AddDesc(err.Error()) } - // 更新到数据库，revision+1 - if err := commonrepo.NewServiceColl().Create(service); err != nil { - log.Errorf("Failed to sync service %s from github path %s error: %v", service.ServiceName, service.SrcPath, err) - return e.ErrCreateTemplate.AddDesc(err.Error()) - } + log.Infof("End of sync service template %s from github path %s", service.ServiceName, service.SrcPath) return nil } diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/github_scanning_task.go b/pkg/microservice/aslan/core/workflow/service/webhook/github_scanning_task.go new file mode 100644 index 0000000000000000000000000000000000000000..e45316552bb0ab5ca0e29fb60b270f20c07fcae5 --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/service/webhook/github_scanning_task.go @@ -0,0 +1,224 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "strconv" + + "github.com/google/go-github/v35/github" + "github.com/hashicorp/go-multierror" + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + scanningservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/testing/service" + "github.com/koderover/zadig/pkg/types" + "go.uber.org/zap" +) + +type gitEventMatcherForScanning interface { + Match(repository *types.ScanningHook) (bool, error) +} + +func TriggerScanningByGithubEvent(event interface{}, requestID string, log *zap.SugaredLogger) error { + //1.find configured testing + scanningList, _, err := commonrepo.NewScanningColl().List(nil, 0, 0) + if err != nil { + log.Errorf("failed to list scanning %v", err) + return err + } + + mErr := &multierror.Error{} + diffSrv := func(pullRequestEvent *github.PullRequestEvent, codehostId int) ([]string, error) { + return findChangedFilesOfPullRequest(pullRequestEvent, codehostId) + } + + log.Infof("Matching scanning list to find matched task to run.") + for _, scanning := range scanningList { + log.Infof("scanning.AdvancedSetting.Hookctl is: [%s], scanning.AdvancedSetting.HookCtl.Enabled is: [%s], scanningName is: [%s]", scanning.AdvancedSetting.HookCtl, scanning.AdvancedSetting.HookCtl.Enabled, scanning.ID) + if scanning.AdvancedSetting.HookCtl != nil && scanning.AdvancedSetting.HookCtl.Enabled { + for _, item := range scanning.AdvancedSetting.HookCtl.Items { + matcher := createGithubEventMatcherForScanning(event, diffSrv, scanning, log) + if matcher == nil { + log.Infof("got a nil matcher for trigger: %s/%s, stopping...", item.RepoOwner, item.RepoName) + continue + } + if matches, err := matcher.Match(item); err != nil { + mErr = multierror.Append(err) + } else if matches { + log.Infof("event match hook %v of %s", item, scanning.Name) + var mergeRequestID string + if ev, isPr := event.(*github.PullRequestEvent); isPr { + mergeRequestID = strconv.Itoa(*ev.PullRequest.Number) + } + triggerRepoInfo := make([]*scanningservice.ScanningRepoInfo, 0) + // for now only one repo is supported + repoInfo := &scanningservice.ScanningRepoInfo{ + CodehostID: item.CodehostID, + Source: item.Source, + RepoOwner: item.RepoOwner, + RepoName: item.RepoName, + Branch: item.Branch, + } + if mergeRequestID != "" { + prID, err := strconv.Atoi(mergeRequestID) + if err != nil { + log.Errorf("failed to convert mergeRequestID: %s to int, error: %s", mergeRequestID, err) + mErr = multierror.Append(mErr, err) + continue + } + repoInfo.PR = prID + } + + triggerRepoInfo = append(triggerRepoInfo, repoInfo) + + if resp, err := scanningservice.CreateScanningTask(scanning.ID.Hex(), triggerRepoInfo, "", "webhook", log); err != nil { + log.Errorf("failed to create testing task when receive event %v due to %v ", event, err) + mErr = multierror.Append(mErr, err) + } else { + log.Infof("succeed to create task %v", resp) + } + } else { + log.Debugf("event not matches %v", item) + } + } + } + } + return mErr.ErrorOrNil() +} + +type githubPushEventMatcherForScanning struct { + log *zap.SugaredLogger + scanning *commonmodels.Scanning + event *github.PushEvent +} + +func (gpem githubPushEventMatcherForScanning) Match(hookRepo *types.ScanningHook) (bool, error) { + ev := gpem.event + if hookRepo == nil { + return false, nil + } + if (hookRepo.RepoOwner + "/" + hookRepo.RepoName) == *ev.Repo.FullName { + matchRepo := ConvertScanningHookToMainHookRepo(hookRepo) + + if !EventConfigured(matchRepo, config.HookEventPush) { + return false, nil + } + + if hookRepo.Branch != getBranchFromRef(*ev.Ref) { + return false, nil + } + + hookRepo.Branch = getBranchFromRef(*ev.Ref) + var changedFiles []string + for _, commit := range ev.Commits { + changedFiles = append(changedFiles, commit.Added...) + changedFiles = append(changedFiles, commit.Removed...) + changedFiles = append(changedFiles, commit.Modified...) + } + + return MatchChanges(matchRepo, changedFiles), nil + } + + return false, nil +} + +type githubMergeEventMatcherForScanning struct { + diffFunc githubPullRequestDiffFunc + log *zap.SugaredLogger + scanning *commonmodels.Scanning + event *github.PullRequestEvent +} + +func (gmem githubMergeEventMatcherForScanning) Match(hookRepo *types.ScanningHook) (bool, error) { + ev := gmem.event + if hookRepo == nil { + return false, nil + } + // TODO: match codehost + if (hookRepo.RepoOwner + "/" + hookRepo.RepoName) == *ev.PullRequest.Base.Repo.FullName { + matchRepo := ConvertScanningHookToMainHookRepo(hookRepo) + if !EventConfigured(matchRepo, config.HookEventPr) { + return false, nil + } + + hookRepo.Branch = *ev.PullRequest.Base.Ref + + if *ev.PullRequest.State == "open" { + var changedFiles []string + changedFiles, err := gmem.diffFunc(ev, hookRepo.CodehostID) + if err != nil { + gmem.log.Warnf("failed to get changes of event %v", ev) + return false, err + } + gmem.log.Debugf("succeed to get %d changes in merge event", len(changedFiles)) + + return MatchChanges(matchRepo, changedFiles), nil + } + + } + return false, nil +} + +type githubTagEventMatcherForScanning struct { + log *zap.SugaredLogger + scanning *commonmodels.Scanning + event *github.CreateEvent +} + +func (gtem githubTagEventMatcherForScanning) Match(hookRepo *types.ScanningHook) (bool, error) { + ev := gtem.event + if (hookRepo.RepoOwner + "/" + hookRepo.RepoName) == *ev.Repo.FullName { + hookInfo := ConvertScanningHookToMainHookRepo(hookRepo) + if !EventConfigured(hookInfo, config.HookEventTag) { + return false, nil + } + + hookRepo.Branch = *ev.Repo.DefaultBranch + + return true, nil + } + + return false, nil +} + +func createGithubEventMatcherForScanning( + event interface{}, diffSrv githubPullRequestDiffFunc, scanning *commonmodels.Scanning, log *zap.SugaredLogger, +) gitEventMatcherForScanning { + switch evt := event.(type) { + case *github.PushEvent: + return &githubPushEventMatcherForScanning{ + scanning: scanning, + log: log, + event: evt, + } + case *github.PullRequestEvent: + return &githubMergeEventMatcherForScanning{ + diffFunc: diffSrv, + log: log, + event: evt, + scanning: scanning, + } + case *github.CreateEvent: + return &githubTagEventMatcherForScanning{ + scanning: scanning, + log: log, + event: evt, + } + } + + return nil +} diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/github_testing_task.go b/pkg/microservice/aslan/core/workflow/service/webhook/github_testing_task.go new file mode 100644 index 0000000000000000000000000000000000000000..8478e0a8d234da68770572650b0d40f0335ab160 --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/service/webhook/github_testing_task.go @@ -0,0 +1,252 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "regexp" + "strconv" + + "github.com/google/go-github/v35/github" + "github.com/hashicorp/go-multierror" + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + testingservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/testing/service" + "github.com/koderover/zadig/pkg/setting" +) + +func TriggerTestByGithubEvent(event interface{}, requestID string, log *zap.SugaredLogger) error { + //1.find configured testing + testingList, err := commonrepo.NewTestingColl().List(&commonrepo.ListTestOption{}) + if err != nil { + log.Errorf("failed to list testing %v", err) + return err + } + mErr := &multierror.Error{} + diffSrv := func(pullRequestEvent *github.PullRequestEvent, codehostId int) ([]string, error) { + return findChangedFilesOfPullRequest(pullRequestEvent, codehostId) + } + for _, testing := range testingList { + if testing.HookCtl != nil && testing.HookCtl.Enabled { + for _, item := range testing.HookCtl.Items { + if item.TestArgs == nil { + continue + } + matcher := createGithubEventMatcherForTesting(event, diffSrv, testing, log) + if matcher == nil { + continue + } + if matches, err := matcher.Match(item.MainRepo); err != nil { + mErr = multierror.Append(err) + } else if matches { + log.Infof("event match hook %v of %s", item.MainRepo, testing.Name) + var mergeRequestID, commitID string + if ev, isPr := event.(*github.PullRequestEvent); isPr { + mergeRequestID = strconv.Itoa(*ev.PullRequest.Number) + commitID = *ev.PullRequest.Head.SHA + } + args := matcher.UpdateTaskArgs(item.TestArgs, requestID) + args.MergeRequestID = mergeRequestID + args.CommitID = commitID + args.Source = setting.SourceFromGithub + args.CodehostID = item.MainRepo.CodehostID + args.RepoOwner = item.MainRepo.RepoOwner + args.RepoName = item.MainRepo.RepoName + if resp, err := testingservice.CreateTestTask(args, log); err != nil { + log.Errorf("failed to create testing task when receive event %v due to %v ", event, err) + mErr = multierror.Append(mErr, err) + } else { + log.Infof("succeed to create task %v", resp) + } + } else { + log.Debugf("event not matches %v", item.MainRepo) + } + } + } + } + return mErr.ErrorOrNil() +} + +type githubPushEventMatcherForTesting struct { + log *zap.SugaredLogger + testing *commonmodels.Testing + event *github.PushEvent +} + +type githubMergeEventMatcherForTesting struct { + diffFunc githubPullRequestDiffFunc + log *zap.SugaredLogger + testing *commonmodels.Testing + event *github.PullRequestEvent +} + +func (gpem *githubPushEventMatcherForTesting) Match(hookRepo *commonmodels.MainHookRepo) (bool, error) { + ev := gpem.event + if !checkRepoNamespaceMatch(hookRepo, *ev.Repo.FullName) { + return false, nil + } + if !EventConfigured(hookRepo, config.HookEventPush) { + return false, nil + } + isRegular := hookRepo.IsRegular + if !isRegular && hookRepo.Branch != getBranchFromRef(*ev.Ref) { + return false, nil + } + + if isRegular { + if matched, _ := regexp.MatchString(hookRepo.Branch, getBranchFromRef(*ev.Ref)); !matched { + return false, nil + } + } + hookRepo.Branch = getBranchFromRef(*ev.Ref) + var changedFiles []string + for _, commit := range ev.Commits { + changedFiles = append(changedFiles, commit.Added...) + changedFiles = append(changedFiles, commit.Removed...) + changedFiles = append(changedFiles, commit.Modified...) + } + + return MatchChanges(hookRepo, changedFiles), nil +} + +func (gpem *githubPushEventMatcherForTesting) UpdateTaskArgs(args *commonmodels.TestTaskArgs, requestID string) *commonmodels.TestTaskArgs { + factory := &testArgsFactory{ + testing: gpem.testing, + reqID: requestID, + } + + factory.Update(args) + return args +} + +type githubTagEventMatcherForTesting struct { + log *zap.SugaredLogger + testing *commonmodels.Testing + event *github.CreateEvent +} + +func (gtem githubTagEventMatcherForTesting) Match(hookRepo *commonmodels.MainHookRepo) (bool, error) { + ev := gtem.event + if !checkRepoNamespaceMatch(hookRepo, *ev.Repo.FullName) { + return false, nil + } + if !EventConfigured(hookRepo, config.HookEventTag) { + return false, nil + } + isRegular := hookRepo.IsRegular + if !isRegular && hookRepo.Branch != *ev.Repo.DefaultBranch { + return false, nil + } + + if isRegular { + if matched, _ := regexp.MatchString(hookRepo.Branch, *ev.Repo.DefaultBranch); !matched { + return false, nil + } + } + hookRepo.Branch = *ev.Repo.DefaultBranch + + return true, nil +} + +func (gtem githubTagEventMatcherForTesting) UpdateTaskArgs(args *commonmodels.TestTaskArgs, requestID string) *commonmodels.TestTaskArgs { + factory := &testArgsFactory{ + testing: gtem.testing, + reqID: requestID, + } + + factory.Update(args) + return args +} + +func createGithubEventMatcherForTesting( + event interface{}, diffSrv githubPullRequestDiffFunc, testing *commonmodels.Testing, log *zap.SugaredLogger, +) gitEventMatcherForTesting { + switch evt := event.(type) { + case *github.PushEvent: + return &githubPushEventMatcherForTesting{ + testing: testing, + log: log, + event: evt, + } + case *github.PullRequestEvent: + return &githubMergeEventMatcherForTesting{ + diffFunc: diffSrv, + log: log, + event: evt, + testing: testing, + } + case *github.CreateEvent: + return &githubTagEventMatcherForTesting{ + testing: testing, + log: log, + event: evt, + } + } + + return nil +} + +func (gmem *githubMergeEventMatcherForTesting) Match(hookRepo *commonmodels.MainHookRepo) (bool, error) { + ev := gmem.event + // TODO: match codehost + + if !checkRepoNamespaceMatch(hookRepo, *ev.PullRequest.Base.Repo.FullName) { + return false, nil + } + + if !EventConfigured(hookRepo, config.HookEventPr) { + return false, nil + } + + isRegular := hookRepo.IsRegular + if !isRegular && hookRepo.Branch != *ev.PullRequest.Base.Ref { + return false, nil + } + + if isRegular { + if matched, _ := regexp.MatchString(hookRepo.Branch, *ev.PullRequest.Base.Ref); !matched { + return false, nil + } + } + hookRepo.Branch = *ev.PullRequest.Base.Ref + + if *ev.PullRequest.State == "open" { + var changedFiles []string + changedFiles, err := gmem.diffFunc(ev, hookRepo.CodehostID) + if err != nil { + gmem.log.Warnf("failed to get changes of event %v", ev) + return false, err + } + gmem.log.Debugf("succeed to get %d changes in merge event", len(changedFiles)) + + return MatchChanges(hookRepo, changedFiles), nil + } + return false, nil +} + +func (gmem *githubMergeEventMatcherForTesting) UpdateTaskArgs(args *commonmodels.TestTaskArgs, requestID string) *commonmodels.TestTaskArgs { + factory := &testArgsFactory{ + testing: gmem.testing, + reqID: requestID, + } + + args = factory.Update(args) + + return args +} diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/github_workflow_task.go b/pkg/microservice/aslan/core/workflow/service/webhook/github_workflow_task.go index a90c906fe6b349ff7316c76008aa13f4d3292034..ed7faffdfad219e92117cf9b418c609b03bbf62d 100644 --- a/pkg/microservice/aslan/core/workflow/service/webhook/github_workflow_task.go +++ b/pkg/microservice/aslan/core/workflow/service/webhook/github_workflow_task.go @@ -19,6 +19,7 @@ package webhook import ( "context" "fmt" + "regexp" "strconv" "strings" @@ -29,12 +30,12 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" git "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/github" workflowservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/codehost" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" "github.com/koderover/zadig/pkg/types" - "github.com/koderover/zadig/pkg/types/permission" ) const SplitSymbol = "&" @@ -42,8 +43,8 @@ const SplitSymbol = "&" type githubPullRequestDiffFunc func(event *github.PullRequestEvent, id int) ([]string, error) type gitEventMatcher interface { - Match(commonmodels.MainHookRepo) (bool, error) - UpdateTaskArgs(*commonmodels.Product, *commonmodels.WorkflowTaskArgs, commonmodels.MainHookRepo, string) *commonmodels.WorkflowTaskArgs + Match(*commonmodels.MainHookRepo) (bool, error) + UpdateTaskArgs(*commonmodels.Product, *commonmodels.WorkflowTaskArgs, *commonmodels.MainHookRepo, string) *commonmodels.WorkflowTaskArgs } type githubPushEventMatcher struct { @@ -52,21 +53,36 @@ type githubPushEventMatcher struct { event *github.PushEvent } -func (gpem *githubPushEventMatcher) Match(hookRepo commonmodels.MainHookRepo) (bool, error) { +func (gpem *githubPushEventMatcher) Match(hookRepo *commonmodels.MainHookRepo) (bool, error) { ev := gpem.event - if (hookRepo.RepoOwner + "/" + hookRepo.RepoName) == *ev.Repo.FullName { - if hookRepo.Branch == getBranchFromRef(*ev.Ref) && EventConfigured(hookRepo, config.HookEventPush) { - var changedFiles []string - for _, commit := range ev.Commits { - changedFiles = append(changedFiles, commit.Added...) - changedFiles = append(changedFiles, commit.Removed...) - changedFiles = append(changedFiles, commit.Modified...) - } - return MatchChanges(hookRepo, changedFiles), nil - } + + if !checkRepoNamespaceMatch(hookRepo, *ev.Repo.FullName) { + return false, nil } - return false, nil + if !EventConfigured(hookRepo, config.HookEventPush) { + return false, nil + } + + isRegular := hookRepo.IsRegular + if !isRegular && hookRepo.Branch != getBranchFromRef(*ev.Ref) { + return false, nil + } + if isRegular { + // Do not use regexp.MustCompile to avoid panic + if matched, _ := regexp.MatchString(hookRepo.Branch, getBranchFromRef(*ev.Ref)); !matched { + return false, nil + } + } + hookRepo.Branch = getBranchFromRef(*ev.Ref) + hookRepo.Committer = *ev.Pusher.Name + var changedFiles []string + for _, commit := range ev.Commits { + changedFiles = append(changedFiles, commit.Added...) + changedFiles = append(changedFiles, commit.Removed...) + changedFiles = append(changedFiles, commit.Modified...) + } + return MatchChanges(hookRepo, changedFiles), nil } func getBranchFromRef(ref string) string { @@ -78,8 +94,16 @@ func getBranchFromRef(ref string) string { return ref } +func getTagFromRef(ref string) string { + prefix := "refs/tags/" + if strings.HasPrefix(ref, prefix) { + return ref[len(prefix):] + } + + return ref +} func (gpem *githubPushEventMatcher) UpdateTaskArgs( - product *commonmodels.Product, args *commonmodels.WorkflowTaskArgs, hookRepo commonmodels.MainHookRepo, requestID string, + product *commonmodels.Product, args *commonmodels.WorkflowTaskArgs, hookRepo *commonmodels.MainHookRepo, requestID string, ) *commonmodels.WorkflowTaskArgs { factory := &workflowArgsFactory{ workflow: gpem.workflow, @@ -87,10 +111,11 @@ func (gpem *githubPushEventMatcher) UpdateTaskArgs( } factory.Update(product, args, &types.Repository{ - CodehostID: hookRepo.CodehostID, - RepoName: hookRepo.RepoName, - RepoOwner: hookRepo.RepoOwner, - Branch: hookRepo.Branch, + CodehostID: hookRepo.CodehostID, + RepoName: hookRepo.RepoName, + RepoOwner: hookRepo.RepoOwner, + RepoNamespace: hookRepo.GetRepoNamespace(), + Branch: hookRepo.Branch, }) return args @@ -103,28 +128,45 @@ type githubMergeEventMatcher struct { event *github.PullRequestEvent } -func (gmem *githubMergeEventMatcher) Match(hookRepo commonmodels.MainHookRepo) (bool, error) { +func (gmem *githubMergeEventMatcher) Match(hookRepo *commonmodels.MainHookRepo) (bool, error) { ev := gmem.event - if (hookRepo.RepoOwner + "/" + hookRepo.RepoName) == *ev.PullRequest.Base.Repo.FullName { - if EventConfigured(hookRepo, config.HookEventPr) && (hookRepo.Branch == *ev.PullRequest.Base.Ref) { - if *ev.PullRequest.State == "open" { - var changedFiles []string - changedFiles, err := gmem.diffFunc(ev, hookRepo.CodehostID) - if err != nil { - gmem.log.Warnf("failed to get changes of event %v", ev) - return false, err - } - gmem.log.Debugf("succeed to get %d changes in merge event", len(changedFiles)) - return MatchChanges(hookRepo, changedFiles), nil - } + if !checkRepoNamespaceMatch(hookRepo, *ev.PullRequest.Base.Repo.FullName) { + return false, nil + } + + if !EventConfigured(hookRepo, config.HookEventPr) { + return false, nil + } + + isRegular := hookRepo.IsRegular + if !isRegular && hookRepo.Branch != *ev.PullRequest.Base.Ref { + return false, nil + } + if isRegular { + if matched, _ := regexp.MatchString(hookRepo.Branch, *ev.PullRequest.Base.Ref); !matched { + return false, nil + } + } + hookRepo.Branch = *ev.PullRequest.Base.Ref + hookRepo.Committer = *ev.PullRequest.User.Login + if *ev.PullRequest.State == "open" { + var changedFiles []string + changedFiles, err := gmem.diffFunc(ev, hookRepo.CodehostID) + if err != nil { + gmem.log.Warnf("failed to get changes of event %v", ev) + return false, err } + gmem.log.Debugf("succeed to get %d changes in merge event", len(changedFiles)) + + return MatchChanges(hookRepo, changedFiles), nil } + return false, nil } func (gmem *githubMergeEventMatcher) UpdateTaskArgs( - product *commonmodels.Product, args *commonmodels.WorkflowTaskArgs, hookRepo commonmodels.MainHookRepo, requestID string, + product *commonmodels.Product, args *commonmodels.WorkflowTaskArgs, hookRepo *commonmodels.MainHookRepo, requestID string, ) *commonmodels.WorkflowTaskArgs { factory := &workflowArgsFactory{ workflow: gmem.workflow, @@ -132,11 +174,12 @@ func (gmem *githubMergeEventMatcher) UpdateTaskArgs( } args = factory.Update(product, args, &types.Repository{ - CodehostID: hookRepo.CodehostID, - RepoName: hookRepo.RepoName, - RepoOwner: hookRepo.RepoOwner, - Branch: hookRepo.Branch, - PR: *gmem.event.PullRequest.Number, + CodehostID: hookRepo.CodehostID, + RepoName: hookRepo.RepoName, + RepoOwner: hookRepo.RepoOwner, + RepoNamespace: hookRepo.GetRepoNamespace(), + Branch: hookRepo.Branch, + PR: *gmem.event.PullRequest.Number, }) return args @@ -145,12 +188,13 @@ func (gmem *githubMergeEventMatcher) UpdateTaskArgs( type workflowArgsFactory struct { workflow *commonmodels.Workflow reqID string + IsYaml bool } func (waf *workflowArgsFactory) Update(product *commonmodels.Product, args *commonmodels.WorkflowTaskArgs, repo *types.Repository) *commonmodels.WorkflowTaskArgs { workflow := waf.workflow args.WorkflowName = workflow.Name - args.WorklowTaskCreator = setting.WebhookTaskCreator + args.WorkflowTaskCreator = setting.WebhookTaskCreator args.ProductTmplName = workflow.ProductTmplName args.ReqID = waf.reqID var targetMap map[string][]commonmodels.DeployEnv @@ -164,7 +208,10 @@ func (waf *workflowArgsFactory) Update(product *commonmodels.Product, args *comm target.Build.Repos = append(target.Build.Repos, repo) if len(target.Deploy) == 0 { if targetMap == nil { - targetMap = getProductTargetMap(product) + targetMap, _ = commonservice.GetProductTargetMap(product) + } + if waf.IsYaml { + targetMap = make(map[string][]commonmodels.DeployEnv) } serviceModuleTarget := fmt.Sprintf("%s%s%s%s%s", args.ProductTmplName, SplitSymbol, target.ServiceName, SplitSymbol, target.Name) target.Deploy = targetMap[serviceModuleTarget] @@ -175,7 +222,7 @@ func (waf *workflowArgsFactory) Update(product *commonmodels.Product, args *comm target.Builds = append(target.Builds, repo) } - if workflow.TestStage != nil && workflow.TestStage.Enabled { + if !waf.IsYaml && workflow.TestStage != nil && workflow.TestStage.Enabled { testArgs := make([]*commonmodels.TestArgs, 0) for _, testName := range workflow.TestStage.TestNames { testArgs = append(testArgs, &commonmodels.TestArgs{ @@ -197,6 +244,57 @@ func (waf *workflowArgsFactory) Update(product *commonmodels.Product, args *comm return args } +type githubTagEventMatcher struct { + log *zap.SugaredLogger + workflow *commonmodels.Workflow + event *github.CreateEvent +} + +func (gtem githubTagEventMatcher) Match(hookRepo *commonmodels.MainHookRepo) (bool, error) { + ev := gtem.event + + if !checkRepoNamespaceMatch(hookRepo, *ev.Repo.FullName) { + return false, nil + } + + if !EventConfigured(hookRepo, config.HookEventTag) { + return false, nil + } + + isRegular := hookRepo.IsRegular + if !isRegular && hookRepo.Branch != *ev.Repo.DefaultBranch { + return false, nil + } + if isRegular { + // Do not use regexp.MustCompile to avoid panic + if matched, _ := regexp.MatchString(hookRepo.Branch, *ev.Repo.DefaultBranch); !matched { + return false, nil + } + } + hookRepo.Tag = getTagFromRef(*ev.Ref) + hookRepo.Committer = *ev.Sender.Name + + return true, nil +} + +func (gtem githubTagEventMatcher) UpdateTaskArgs(product *commonmodels.Product, args *commonmodels.WorkflowTaskArgs, hookRepo *commonmodels.MainHookRepo, requestID string) *commonmodels.WorkflowTaskArgs { + factory := &workflowArgsFactory{ + workflow: gtem.workflow, + reqID: requestID, + } + + factory.Update(product, args, &types.Repository{ + CodehostID: hookRepo.CodehostID, + RepoName: hookRepo.RepoName, + RepoOwner: hookRepo.RepoOwner, + RepoNamespace: hookRepo.GetRepoNamespace(), + Branch: hookRepo.Branch, + Tag: hookRepo.Tag, + }) + + return args +} + func createGithubEventMatcher( event interface{}, diffSrv githubPullRequestDiffFunc, workflow *commonmodels.Workflow, log *zap.SugaredLogger, ) gitEventMatcher { @@ -214,6 +312,12 @@ func createGithubEventMatcher( event: evt, workflow: workflow, } + case *github.CreateEvent: + return &githubTagEventMatcher{ + workflow: workflow, + log: log, + event: evt, + } } return nil @@ -294,11 +398,13 @@ func TriggerWorkflowByGithubEvent(event interface{}, baseURI, deliveryID, reques args.Source = setting.SourceFromGithub args.CodehostID = item.MainRepo.CodehostID args.RepoOwner = item.MainRepo.RepoOwner + args.RepoNamespace = item.MainRepo.GetRepoNamespace() args.RepoName = item.MainRepo.RepoName + args.Committer = item.MainRepo.Committer args.HookPayload = hookPayload // 3. create task with args - if resp, err := workflowservice.CreateWorkflowTask(args, setting.WebhookTaskCreator, permission.AnonymousUserID, false, log); err != nil { + if resp, err := workflowservice.CreateWorkflowTask(args, setting.WebhookTaskCreator, log); err != nil { log.Errorf("failed to create workflow task when receive push event due to %v ", err) mErr = multierror.Append(mErr, err) } else { @@ -315,12 +421,12 @@ func TriggerWorkflowByGithubEvent(event interface{}, baseURI, deliveryID, reques } func findChangedFilesOfPullRequest(event *github.PullRequestEvent, codehostID int) ([]string, error) { - detail, err := codehost.GetCodehostDetail(codehostID) + detail, err := systemconfig.New().GetCodeHost(codehostID) if err != nil { return nil, fmt.Errorf("failed to find codehost %d: %v", codehostID, err) } //pullrequest文件修改 - githubCli := git.NewClient(detail.OauthToken, config.ProxyHTTPSAddr()) + githubCli := git.NewClient(detail.AccessToken, config.ProxyHTTPSAddr(), detail.EnableProxy) commitComparison, _, err := githubCli.Repositories.CompareCommits(context.Background(), *event.PullRequest.Base.Repo.Owner.Login, *event.PullRequest.Base.Repo.Name, *event.PullRequest.Base.SHA, *event.PullRequest.Head.SHA) if err != nil { return nil, fmt.Errorf("failed to get changes from github, err: %v", err) diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/gitlab.go b/pkg/microservice/aslan/core/workflow/service/webhook/gitlab.go index 390139988c0915ed7c491375c5e4f34efb594134..da9380f6da84179253f13fd2c66ef5ebeb9fe146 100644 --- a/pkg/microservice/aslan/core/workflow/service/webhook/gitlab.go +++ b/pkg/microservice/aslan/core/workflow/service/webhook/gitlab.go @@ -17,11 +17,9 @@ limitations under the License. package webhook import ( - "encoding/json" "errors" "fmt" "net/http" - "strings" "sync" "time" @@ -29,9 +27,9 @@ import ( "github.com/xanzy/go-gitlab" "go.uber.org/zap" + "github.com/koderover/zadig/pkg/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/collie" gitservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/git" "github.com/koderover/zadig/pkg/setting" e "github.com/koderover/zadig/pkg/tool/errors" @@ -61,13 +59,10 @@ func ProcessGitlabHook(payload []byte, req *http.Request, requestID string, log return err } - forwardedProto := req.Header.Get("X-Forwarded-Proto") - forwardedHost := req.Header.Get("X-Forwarded-Host") - baseURI := fmt.Sprintf("%s://%s", forwardedProto, forwardedHost) - - var eventPush *EventPush + baseURI := config.SystemAddress() var pushEvent *gitlab.PushEvent var mergeEvent *gitlab.MergeEvent + var tagEvent *gitlab.TagEvent var errorList = &multierror.Error{} switch event.(type) { @@ -84,27 +79,24 @@ func ProcessGitlabHook(payload []byte, req *http.Request, requestID string, log } } - go collie.CallGitlabWebHook(forwardedProto, forwardedHost, payload, string(eventType), log) - switch event := event.(type) { case *gitlab.PushEvent: - eventPush = &EventPush{ - Before: event.Before, - After: event.After, - Ref: event.Ref, - CheckoutSha: event.CheckoutSHA, - ProjectID: event.ProjectID, - Body: string(payload), - } pushEvent = event - case *gitlab.MergeEvent: - mergeEvent = event - } - //触发更新服务模板webhook - if eventPush != nil { - if err = updateServiceTemplateByPushEvent(eventPush, log); err != nil { + changeFiles := make([]string, 0) + for _, commit := range pushEvent.Commits { + changeFiles = append(changeFiles, commit.Added...) + changeFiles = append(changeFiles, commit.Removed...) + changeFiles = append(changeFiles, commit.Modified...) + } + pathWithNamespace := pushEvent.Project.PathWithNamespace + // trigger service template to re-sync from remote repo + if err = updateServiceTemplateByPushEvent(changeFiles, pathWithNamespace, log); err != nil { errorList = multierror.Append(errorList, err) } + case *gitlab.MergeEvent: + mergeEvent = event + case *gitlab.TagEvent: + tagEvent = event } //触发工作流webhook和测试管理webhook @@ -149,6 +141,14 @@ func ProcessGitlabHook(payload []byte, req *http.Request, requestID string, log errorList = multierror.Append(errorList, err) } }() + + wg.Add(1) + go func() { + defer wg.Done() + if err = TriggerScanningByGitlabEvent(pushEvent, baseURI, requestID, log); err != nil { + errorList = multierror.Append(errorList, err) + } + }() } if mergeEvent != nil { @@ -178,6 +178,42 @@ func ProcessGitlabHook(payload []byte, req *http.Request, requestID string, log errorList = multierror.Append(errorList, err) } }() + + wg.Add(1) + go func() { + defer wg.Done() + if err = TriggerScanningByGitlabEvent(mergeEvent, baseURI, requestID, log); err != nil { + errorList = multierror.Append(errorList, err) + } + }() + } + + if tagEvent != nil { + // workflow webhook + wg.Add(1) + go func() { + defer wg.Done() + if err = TriggerWorkflowByGitlabEvent(tagEvent, baseURI, requestID, log); err != nil { + errorList = multierror.Append(errorList, err) + } + }() + + //test webhook + wg.Add(1) + go func() { + defer wg.Done() + if err = TriggerTestByGitlabEvent(tagEvent, baseURI, requestID, log); err != nil { + errorList = multierror.Append(errorList, err) + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + if err = TriggerScanningByGitlabEvent(tagEvent, baseURI, requestID, log); err != nil { + errorList = multierror.Append(errorList, err) + } + }() } wg.Wait() @@ -245,32 +281,9 @@ type RepositoryInfo struct { VisibilityLevel int `json:"visibility_level"` } -func updateServiceTemplateByPushEvent(event *EventPush, log *zap.SugaredLogger) error { +func updateServiceTemplateByPushEvent(diffs []string, pathWithNamespace string, log *zap.SugaredLogger) error { log.Infof("EVENT: GITLAB WEBHOOK UPDATING SERVICE TEMPLATE") - gitlabEvent := &GitlabEvent{} - err := json.Unmarshal([]byte(event.Body), gitlabEvent) - if err != nil { - log.Errorf("Get Project ID failed, error: %v", err) - return err - } - - address, err := GetAddress(gitlabEvent.Project.WebURL) - if err != nil { - log.Errorf("GetGitlabAddress failed, error: %v", err) - return err - } - - client, err := getGitlabClientByAddress(address) - if err != nil { - return err - } - - diffs, err := client.Compare(event.ProjectID, event.Before, event.After) - if err != nil { - log.Errorf("Failed to get push event diffs, error: %v", err) - return err - } serviceTmpls, err := GetGitlabServiceTemplates() if err != nil { log.Errorf("Failed to get gitlab service templates, error: %v", err) @@ -280,15 +293,18 @@ func updateServiceTemplateByPushEvent(event *EventPush, log *zap.SugaredLogger) errs := &multierror.Error{} for _, service := range serviceTmpls { - srcPath := service.SrcPath - _, _, _, _, path, _, err := GetOwnerRepoBranchPath(srcPath) + if service.GetRepoNamespace()+"/"+service.RepoName != pathWithNamespace { + continue + } + + path, err := getServiceSrcPath(service) if err != nil { errs = multierror.Append(errs, err) } // 判断PushEvent的Diffs中是否包含该服务模板的src_path affected := false for _, diff := range diffs { - if strings.Contains(diff.OldPath, path) || strings.Contains(diff.NewPath, path) { + if subElem(path, diff) { affected = true break } @@ -312,8 +328,7 @@ func updateServiceTemplateByPushEvent(event *EventPush, log *zap.SugaredLogger) func GetGitlabServiceTemplates() ([]*commonmodels.Service, error) { opt := &commonrepo.ServiceListOption{ - Type: setting.K8SDeployType, - Source: setting.SourceFromGitlab, + Source: setting.SourceFromGitlab, } return commonrepo.NewServiceColl().ListMaxRevisions(opt) } @@ -348,11 +363,6 @@ func SyncServiceTemplateFromGitlab(service *commonmodels.Service, log *zap.Sugar log.Errorf("ensureServiceTmpl error: %+v", err) return e.ErrValidateTemplate.AddDesc(err.Error()) } - // 更新到数据库，revision+1 - if err := commonrepo.NewServiceColl().Create(service); err != nil { - log.Errorf("Failed to sync service %s from gitlab path %s error: %v", service.ServiceName, service.SrcPath, err) - return e.ErrCreateTemplate.AddDesc(err.Error()) - } log.Infof("End of sync service template %s from gitlab path %s", service.ServiceName, service.SrcPath) return nil } diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/gitlab_pipeline_task.go b/pkg/microservice/aslan/core/workflow/service/webhook/gitlab_pipeline_task.go index 9370a064e1a31bf029083a142363312e841e215b..d0154307519f350e21058f0a129ddc6698db4e99 100644 --- a/pkg/microservice/aslan/core/workflow/service/webhook/gitlab_pipeline_task.go +++ b/pkg/microservice/aslan/core/workflow/service/webhook/gitlab_pipeline_task.go @@ -46,8 +46,8 @@ func (pgpem *pipelineGitlabPushEventMatcher) Match(hookRepo *commonmodels.MainHo var ( changedFiles = make([]string, 0) ) - if (hookRepo.RepoOwner + "/" + hookRepo.RepoName) == ev.Project.PathWithNamespace { - if hookRepo.Branch == getBranchFromRef(ev.Ref) && EventConfigured(*hookRepo, config.HookEventPush) { + if checkRepoNamespaceMatch(hookRepo, ev.Project.PathWithNamespace) { + if hookRepo.Branch == getBranchFromRef(ev.Ref) && EventConfigured(hookRepo, config.HookEventPush) { for _, commit := range ev.Commits { changedFiles = append(changedFiles, commit.Added...) changedFiles = append(changedFiles, commit.Removed...) @@ -55,7 +55,7 @@ func (pgpem *pipelineGitlabPushEventMatcher) Match(hookRepo *commonmodels.MainHo } } } - if MatchChanges(*hookRepo, changedFiles) { + if MatchChanges(hookRepo, changedFiles) { var ( branch = getBranchFromRef(ev.Ref) ref = ev.Ref @@ -65,6 +65,7 @@ func (pgpem *pipelineGitlabPushEventMatcher) Match(hookRepo *commonmodels.MainHo eventRepo := &types.Repository{ RepoOwner: hookRepo.RepoOwner, + RepoNamespace: hookRepo.GetRepoNamespace(), RepoName: hookRepo.RepoName, Branch: branch, CommitID: commitID, @@ -107,8 +108,8 @@ func (pgmem *pipelineGitlabMergeEventMatcher) Match(hookRepo *commonmodels.MainH err error changedFiles = make([]string, 0) ) - if (hookRepo.RepoOwner + "/" + hookRepo.RepoName) == ev.ObjectAttributes.Target.PathWithNamespace { - if EventConfigured(*hookRepo, config.HookEventPr) && (hookRepo.Branch == ev.ObjectAttributes.TargetBranch) { + if checkRepoNamespaceMatch(hookRepo, ev.ObjectAttributes.Target.PathWithNamespace) { + if EventConfigured(hookRepo, config.HookEventPr) && (hookRepo.Branch == ev.ObjectAttributes.TargetBranch) { if ev.ObjectAttributes.State == "opened" { changedFiles, err = pgmem.diffFunc(ev, hookRepo.CodehostID) if err != nil { @@ -119,7 +120,7 @@ func (pgmem *pipelineGitlabMergeEventMatcher) Match(hookRepo *commonmodels.MainH } } } - if MatchChanges(*hookRepo, changedFiles) { + if MatchChanges(hookRepo, changedFiles) { var ( branch = ev.ObjectAttributes.TargetBranch commitID = ev.ObjectAttributes.LastCommit.ID @@ -128,6 +129,7 @@ func (pgmem *pipelineGitlabMergeEventMatcher) Match(hookRepo *commonmodels.MainH eventRepo := &types.Repository{ RepoOwner: hookRepo.RepoOwner, + RepoNamespace: hookRepo.GetRepoNamespace(), RepoName: hookRepo.RepoName, PR: ev.ObjectAttributes.IID, Branch: branch, @@ -226,7 +228,7 @@ func TriggerPipelineByGitlabEvent(event interface{}, baseURI, requestID string, hookRepo.RepoOwner = taskargs.HookPayload.Owner if notification == nil { notification, _ = scmnotify.NewService().SendInitWebhookComment( - hookRepo, taskargs.Builds[0].PR, baseURI, true, false, log, + hookRepo, taskargs.Builds[0].PR, baseURI, true, false, false, log, ) } } diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/gitlab_scanning_task.go b/pkg/microservice/aslan/core/workflow/service/webhook/gitlab_scanning_task.go new file mode 100644 index 0000000000000000000000000000000000000000..957e5df0f72d924ae81705349d29b2dd1a5aa287 --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/service/webhook/gitlab_scanning_task.go @@ -0,0 +1,228 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "github.com/xanzy/go-gitlab" + "go.uber.org/zap" + + "github.com/hashicorp/go-multierror" + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/scmnotify" + scanningservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/testing/service" + "github.com/koderover/zadig/pkg/types" +) + +func TriggerScanningByGitlabEvent(event interface{}, baseURI, requestID string, log *zap.SugaredLogger) error { + // 1. find configured testing + scanningList, _, err := commonrepo.NewScanningColl().List(nil, 0, 0) + if err != nil { + log.Errorf("failed to list testing %v", err) + return err + } + + mErr := &multierror.Error{} + diffSrv := func(mergeEvent *gitlab.MergeEvent, codehostId int) ([]string, error) { + return findChangedFilesOfMergeRequest(mergeEvent, codehostId) + } + + var notification *commonmodels.Notification + + for _, scanning := range scanningList { + if scanning.AdvancedSetting.HookCtl != nil && scanning.AdvancedSetting.HookCtl.Enabled { + log.Infof("find %d hooks in scanning %s", len(scanning.AdvancedSetting.HookCtl.Items), scanning.Name) + for _, item := range scanning.AdvancedSetting.HookCtl.Items { + // 2. match webhook + matcher := createGitlabEventMatcherForScanning(event, diffSrv, scanning, log) + if matcher == nil { + continue + } + + if matches, err := matcher.Match(item); err != nil { + mErr = multierror.Append(mErr, err) + } else if matches { + log.Infof("event match hook %v of %s", item, scanning.Name) + var mergeRequestID int + if ev, isPr := event.(*gitlab.MergeEvent); isPr { + // 如果是merge request，且该webhook触发器配置了自动取消， + // 则需要确认该merge request在本次commit之前的commit触发的任务是否处理完，没有处理完则取消掉。 + mergeRequestID = ev.ObjectAttributes.IID + + if notification == nil { + mainRepo := ConvertScanningHookToMainHookRepo(item) + notification, _ = scmnotify.NewService().SendInitWebhookComment( + mainRepo, ev.ObjectAttributes.IID, baseURI, false, false, true, log, + ) + } + } + + triggerRepoInfo := make([]*scanningservice.ScanningRepoInfo, 0) + // for now only one repo is supported + repoInfo := &scanningservice.ScanningRepoInfo{ + CodehostID: item.CodehostID, + Source: item.Source, + RepoOwner: item.RepoOwner, + RepoName: item.RepoName, + Branch: item.Branch, + } + triggerRepoInfo = append(triggerRepoInfo, repoInfo) + + repoInfo.PR = mergeRequestID + + notificationID := "" + if notification != nil { + notificationID = notification.ID.Hex() + } + + if resp, err := scanningservice.CreateScanningTask(scanning.ID.Hex(), triggerRepoInfo, notificationID, "webhook", log); err != nil { + log.Errorf("failed to create testing task when receive event %v due to %v ", event, err) + mErr = multierror.Append(mErr, err) + } else { + log.Infof("succeed to create task %v", resp) + } + } else { + log.Debugf("event not matches %v", item) + } + } + } + } + + return mErr.ErrorOrNil() +} + +func createGitlabEventMatcherForScanning( + event interface{}, diffSrv gitlabMergeRequestDiffFunc, scanning *commonmodels.Scanning, log *zap.SugaredLogger, +) gitEventMatcherForScanning { + switch evt := event.(type) { + case *gitlab.PushEvent: + return &gitlabPushEventMatcherForScanning{ + scanning: scanning, + log: log, + event: evt, + } + case *gitlab.MergeEvent: + return &gitlabMergeEventMatcherForScanning{ + diffFunc: diffSrv, + log: log, + event: evt, + scanning: scanning, + } + case *gitlab.TagEvent: + return &gitlabTagEventMatcherForScanning{ + scanning: scanning, + log: log, + event: evt, + } + } + + return nil +} + +type gitlabPushEventMatcherForScanning struct { + log *zap.SugaredLogger + scanning *commonmodels.Scanning + event *gitlab.PushEvent +} + +func (gpem *gitlabPushEventMatcherForScanning) Match(hookRepo *types.ScanningHook) (bool, error) { + ev := gpem.event + if hookRepo == nil { + return false, nil + } + if (hookRepo.RepoOwner + "/" + hookRepo.RepoName) == ev.Project.PathWithNamespace { + matchRepo := ConvertScanningHookToMainHookRepo(hookRepo) + + if !EventConfigured(matchRepo, config.HookEventPush) { + return false, nil + } + if hookRepo.Branch != getBranchFromRef(ev.Ref) { + return false, nil + } + + hookRepo.Branch = getBranchFromRef(ev.Ref) + var changedFiles []string + for _, commit := range ev.Commits { + changedFiles = append(changedFiles, commit.Added...) + changedFiles = append(changedFiles, commit.Removed...) + changedFiles = append(changedFiles, commit.Modified...) + } + + return MatchChanges(matchRepo, changedFiles), nil + } + + return false, nil +} + +type gitlabMergeEventMatcherForScanning struct { + diffFunc gitlabMergeRequestDiffFunc + log *zap.SugaredLogger + scanning *commonmodels.Scanning + event *gitlab.MergeEvent +} + +func (gmem *gitlabMergeEventMatcherForScanning) Match(hookRepo *types.ScanningHook) (bool, error) { + ev := gmem.event + if hookRepo == nil { + return false, nil + } + // TODO: match codehost + if (hookRepo.RepoOwner + "/" + hookRepo.RepoName) == ev.ObjectAttributes.Target.PathWithNamespace { + matchRepo := ConvertScanningHookToMainHookRepo(hookRepo) + if !EventConfigured(matchRepo, config.HookEventPr) { + return false, nil + } + + hookRepo.Branch = ev.ObjectAttributes.TargetBranch + + if ev.ObjectAttributes.State == "opened" { + var changedFiles []string + changedFiles, err := gmem.diffFunc(ev, hookRepo.CodehostID) + if err != nil { + gmem.log.Warnf("failed to get changes of event %v", ev) + return false, err + } + gmem.log.Debugf("succeed to get %d changes in merge event", len(changedFiles)) + + return MatchChanges(matchRepo, changedFiles), nil + } + + } + return false, nil +} + +type gitlabTagEventMatcherForScanning struct { + log *zap.SugaredLogger + scanning *commonmodels.Scanning + event *gitlab.TagEvent +} + +func (gtem *gitlabTagEventMatcherForScanning) Match(hookRepo *types.ScanningHook) (bool, error) { + ev := gtem.event + if (hookRepo.RepoOwner + "/" + hookRepo.RepoName) == ev.Project.PathWithNamespace { + hookInfo := ConvertScanningHookToMainHookRepo(hookRepo) + if !EventConfigured(hookInfo, config.HookEventTag) { + return false, nil + } + + hookRepo.Branch = ev.Project.DefaultBranch + return true, nil + } + + return false, nil +} diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/gitlab_testing_task.go b/pkg/microservice/aslan/core/workflow/service/webhook/gitlab_testing_task.go index 082dd8fc9664da18dfe0d72a36b60b497b7535ee..eb50b12cfc8413a6d9d7ec03d88a8900601a6add 100644 --- a/pkg/microservice/aslan/core/workflow/service/webhook/gitlab_testing_task.go +++ b/pkg/microservice/aslan/core/workflow/service/webhook/gitlab_testing_task.go @@ -17,6 +17,7 @@ limitations under the License. package webhook import ( + "regexp" "strconv" multierror "github.com/hashicorp/go-multierror" @@ -32,7 +33,7 @@ import ( ) type gitEventMatcherForTesting interface { - Match(commonmodels.MainHookRepo) (bool, error) + Match(*commonmodels.MainHookRepo) (bool, error) UpdateTaskArgs(*commonmodels.TestTaskArgs, string) *commonmodels.TestTaskArgs } @@ -56,22 +57,32 @@ type gitlabPushEventMatcherForTesting struct { event *gitlab.PushEvent } -func (gpem *gitlabPushEventMatcherForTesting) Match(hookRepo commonmodels.MainHookRepo) (bool, error) { +func (gpem *gitlabPushEventMatcherForTesting) Match(hookRepo *commonmodels.MainHookRepo) (bool, error) { ev := gpem.event - if (hookRepo.RepoOwner + "/" + hookRepo.RepoName) == ev.Project.PathWithNamespace { - if hookRepo.Branch == getBranchFromRef(ev.Ref) && EventConfigured(hookRepo, config.HookEventPush) { - var changedFiles []string - for _, commit := range ev.Commits { - changedFiles = append(changedFiles, commit.Added...) - changedFiles = append(changedFiles, commit.Removed...) - changedFiles = append(changedFiles, commit.Modified...) - } + if !checkRepoNamespaceMatch(hookRepo, ev.Project.PathWithNamespace) { + return false, nil + } + if !EventConfigured(hookRepo, config.HookEventPush) { + return false, nil + } + isRegular := hookRepo.IsRegular + if !isRegular && hookRepo.Branch != getBranchFromRef(ev.Ref) { + return false, nil + } - return MatchChanges(hookRepo, changedFiles), nil + if isRegular { + if matched, _ := regexp.MatchString(hookRepo.Branch, getBranchFromRef(ev.Ref)); !matched { + return false, nil } } - - return false, nil + hookRepo.Branch = getBranchFromRef(ev.Ref) + var changedFiles []string + for _, commit := range ev.Commits { + changedFiles = append(changedFiles, commit.Added...) + changedFiles = append(changedFiles, commit.Removed...) + changedFiles = append(changedFiles, commit.Modified...) + } + return MatchChanges(hookRepo, changedFiles), nil } func (gpem *gitlabPushEventMatcherForTesting) UpdateTaskArgs(args *commonmodels.TestTaskArgs, requestID string) *commonmodels.TestTaskArgs { @@ -84,6 +95,46 @@ func (gpem *gitlabPushEventMatcherForTesting) UpdateTaskArgs(args *commonmodels. return args } +type gitlabTagEventMatcherForTesting struct { + log *zap.SugaredLogger + testing *commonmodels.Testing + event *gitlab.TagEvent +} + +func (gtem gitlabTagEventMatcherForTesting) Match(hookRepo *commonmodels.MainHookRepo) (bool, error) { + ev := gtem.event + + if !checkRepoNamespaceMatch(hookRepo, ev.Project.PathWithNamespace) { + return false, nil + } + + if !EventConfigured(hookRepo, config.HookEventTag) { + return false, nil + } + isRegular := hookRepo.IsRegular + if !isRegular && hookRepo.Branch != ev.Project.DefaultBranch { + return false, nil + } + + if isRegular { + if matched, _ := regexp.MatchString(hookRepo.Branch, ev.Project.DefaultBranch); !matched { + return false, nil + } + } + hookRepo.Branch = ev.Project.DefaultBranch + return true, nil +} + +func (gtem gitlabTagEventMatcherForTesting) UpdateTaskArgs(args *commonmodels.TestTaskArgs, requestID string) *commonmodels.TestTaskArgs { + factory := &testArgsFactory{ + testing: gtem.testing, + reqID: requestID, + } + + factory.Update(args) + return args +} + // TriggerTestByGitlabEvent 测试管理模块的触发器任务 func TriggerTestByGitlabEvent(event interface{}, baseURI, requestID string, log *zap.SugaredLogger) error { // 1. find configured testing @@ -139,7 +190,7 @@ func TriggerTestByGitlabEvent(event interface{}, baseURI, requestID string, log // 发送本次commit的通知 if notification == nil { notification, _ = scmnotify.NewService().SendInitWebhookComment( - &item.MainRepo, ev.ObjectAttributes.IID, baseURI, false, true, log, + item.MainRepo, ev.ObjectAttributes.IID, baseURI, false, true, false, log, ) } } @@ -154,6 +205,7 @@ func TriggerTestByGitlabEvent(event interface{}, baseURI, requestID string, log args.Source = setting.SourceFromGitlab args.CodehostID = item.MainRepo.CodehostID args.RepoOwner = item.MainRepo.RepoOwner + args.RepoNamespace = item.MainRepo.GetRepoNamespace() args.RepoName = item.MainRepo.RepoName // 3. create task with args @@ -190,6 +242,12 @@ func createGitlabEventMatcherForTesting( event: evt, testing: testing, } + case *gitlab.TagEvent: + return &gitlabTagEventMatcherForTesting{ + testing: testing, + log: log, + event: evt, + } } return nil @@ -202,24 +260,40 @@ type gitlabMergeEventMatcherForTesting struct { event *gitlab.MergeEvent } -func (gmem *gitlabMergeEventMatcherForTesting) Match(hookRepo commonmodels.MainHookRepo) (bool, error) { +func (gmem *gitlabMergeEventMatcherForTesting) Match(hookRepo *commonmodels.MainHookRepo) (bool, error) { ev := gmem.event // TODO: match codehost - if (hookRepo.RepoOwner + "/" + hookRepo.RepoName) == ev.ObjectAttributes.Target.PathWithNamespace { - if EventConfigured(hookRepo, config.HookEventPr) && (hookRepo.Branch == ev.ObjectAttributes.TargetBranch) { - if ev.ObjectAttributes.State == "opened" { - var changedFiles []string - changedFiles, err := gmem.diffFunc(ev, hookRepo.CodehostID) - if err != nil { - gmem.log.Warnf("failed to get changes of event %v", ev) - return false, err - } - gmem.log.Debugf("succeed to get %d changes in merge event", len(changedFiles)) + if !checkRepoNamespaceMatch(hookRepo, ev.ObjectAttributes.Target.PathWithNamespace) { + return false, nil + } - return MatchChanges(hookRepo, changedFiles), nil - } + if !EventConfigured(hookRepo, config.HookEventPr) { + return false, nil + } + + isRegular := hookRepo.IsRegular + if !isRegular && hookRepo.Branch != ev.ObjectAttributes.TargetBranch { + return false, nil + } + + if isRegular { + if matched, _ := regexp.MatchString(hookRepo.Branch, ev.ObjectAttributes.TargetBranch); !matched { + return false, nil } } + hookRepo.Branch = ev.ObjectAttributes.TargetBranch + + if ev.ObjectAttributes.State == "opened" { + var changedFiles []string + changedFiles, err := gmem.diffFunc(ev, hookRepo.CodehostID) + if err != nil { + gmem.log.Warnf("failed to get changes of event %v", ev) + return false, err + } + gmem.log.Debugf("succeed to get %d changes in merge event", len(changedFiles)) + + return MatchChanges(hookRepo, changedFiles), nil + } return false, nil } diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/gitlab_workflow_task.go b/pkg/microservice/aslan/core/workflow/service/webhook/gitlab_workflow_task.go index a58c4f5c115a0aadb5d98e9a9c56cea7f56114a7..12c9b5269b51d31a539a61003c88117a33e81af5 100644 --- a/pkg/microservice/aslan/core/workflow/service/webhook/gitlab_workflow_task.go +++ b/pkg/microservice/aslan/core/workflow/service/webhook/gitlab_workflow_task.go @@ -17,7 +17,9 @@ limitations under the License. package webhook import ( + "encoding/json" "fmt" + "regexp" "strconv" "strings" "sync" @@ -26,6 +28,7 @@ import ( "github.com/hashicorp/go-multierror" "github.com/xanzy/go-gitlab" "go.uber.org/zap" + "gopkg.in/yaml.v3" "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" @@ -35,66 +38,112 @@ import ( environmentservice "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" workflowservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/codehost" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" e "github.com/koderover/zadig/pkg/tool/errors" gitlabtool "github.com/koderover/zadig/pkg/tool/git/gitlab" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/types" - "github.com/koderover/zadig/pkg/types/permission" "github.com/koderover/zadig/pkg/util" ) type gitlabMergeRequestDiffFunc func(event *gitlab.MergeEvent, id int) ([]string, error) type gitlabMergeEventMatcher struct { - diffFunc gitlabMergeRequestDiffFunc - log *zap.SugaredLogger - workflow *commonmodels.Workflow - event *gitlab.MergeEvent + diffFunc gitlabMergeRequestDiffFunc + log *zap.SugaredLogger + workflow *commonmodels.Workflow + event *gitlab.MergeEvent + trigger *TriggerYaml + isYaml bool + yamlServiceChanged []BuildServices } -func (gmem *gitlabMergeEventMatcher) Match(hookRepo commonmodels.MainHookRepo) (bool, error) { +func (gmem *gitlabMergeEventMatcher) Match(hookRepo *commonmodels.MainHookRepo) (bool, error) { ev := gmem.event // TODO: match codehost - if (hookRepo.RepoOwner + "/" + hookRepo.RepoName) == ev.ObjectAttributes.Target.PathWithNamespace { - if EventConfigured(hookRepo, config.HookEventPr) && (hookRepo.Branch == ev.ObjectAttributes.TargetBranch) { - if ev.ObjectAttributes.State == "opened" { - var changedFiles []string - changedFiles, err := gmem.diffFunc(ev, hookRepo.CodehostID) - if err != nil { - gmem.log.Warnf("failed to get changes of event %v, err:%v", ev, err) - return false, err - } - gmem.log.Debugf("succeed to get %d changes in merge event", len(changedFiles)) - - return MatchChanges(hookRepo, changedFiles), nil + if !checkRepoNamespaceMatch(hookRepo, ev.ObjectAttributes.Target.PathWithNamespace) { + return false, nil + } + if !EventConfigured(hookRepo, config.HookEventPr) { + return false, nil + } + if gmem.isYaml { + refFlag := false + for _, ref := range gmem.trigger.Rules.Branchs { + if matched, _ := regexp.MatchString(ref, getBranchFromRef(ev.ObjectAttributes.TargetBranch)); matched { + refFlag = true + break + } + } + if !refFlag { + return false, nil + } + } else { + isRegular := hookRepo.IsRegular + if !isRegular && hookRepo.Branch != ev.ObjectAttributes.TargetBranch { + return false, nil + } + if isRegular { + if matched, _ := regexp.MatchString(hookRepo.Branch, ev.ObjectAttributes.TargetBranch); !matched { + return false, nil } } } + hookRepo.Branch = ev.ObjectAttributes.TargetBranch + hookRepo.Committer = ev.User.Username + if ev.ObjectAttributes.State == "opened" { + var changedFiles []string + changedFiles, err := gmem.diffFunc(ev, hookRepo.CodehostID) + if err != nil { + gmem.log.Warnf("failed to get changes of event %v, err:%s", ev, err) + return false, err + } + gmem.log.Debugf("succeed to get %d changes in merge event", len(changedFiles)) + if gmem.isYaml { + serviceChangeds := ServicesMatchChangesFiles(gmem.trigger.Rules.MatchFolders, changedFiles) + gmem.yamlServiceChanged = serviceChangeds + return len(serviceChangeds) != 0, nil + } + return MatchChanges(hookRepo, changedFiles), nil + } return false, nil } func (gmem *gitlabMergeEventMatcher) UpdateTaskArgs( - product *commonmodels.Product, args *commonmodels.WorkflowTaskArgs, hookRepo commonmodels.MainHookRepo, requestID string, + product *commonmodels.Product, args *commonmodels.WorkflowTaskArgs, hookRepo *commonmodels.MainHookRepo, requestID string, ) *commonmodels.WorkflowTaskArgs { + if gmem.isYaml { + var targets []*commonmodels.TargetArgs + for _, target := range args.Target { + for _, bs := range gmem.yamlServiceChanged { + if target.Name == bs.ServiceModule && target.ServiceName == bs.Name { + targets = append(targets, target) + break + } + } + } + args.Target = targets + } factory := &workflowArgsFactory{ workflow: gmem.workflow, reqID: requestID, + IsYaml: gmem.isYaml, } args = factory.Update(product, args, &types.Repository{ - CodehostID: hookRepo.CodehostID, - RepoName: hookRepo.RepoName, - RepoOwner: hookRepo.RepoOwner, - Branch: hookRepo.Branch, - PR: gmem.event.ObjectAttributes.IID, + CodehostID: hookRepo.CodehostID, + RepoName: hookRepo.RepoName, + RepoOwner: hookRepo.RepoOwner, + RepoNamespace: hookRepo.GetRepoNamespace(), + Branch: hookRepo.Branch, + PR: gmem.event.ObjectAttributes.IID, }) return args } func createGitlabEventMatcher( - event interface{}, diffSrv gitlabMergeRequestDiffFunc, workflow *commonmodels.Workflow, log *zap.SugaredLogger, + event interface{}, diffSrv gitlabMergeRequestDiffFunc, workflow *commonmodels.Workflow, isyaml bool, trigger *TriggerYaml, log *zap.SugaredLogger, ) gitEventMatcher { switch evt := event.(type) { case *gitlab.PushEvent: @@ -102,6 +151,8 @@ func createGitlabEventMatcher( workflow: workflow, log: log, event: evt, + trigger: trigger, + isYaml: isyaml, } case *gitlab.MergeEvent: return &gitlabMergeEventMatcher{ @@ -109,6 +160,16 @@ func createGitlabEventMatcher( log: log, event: evt, workflow: workflow, + trigger: trigger, + isYaml: isyaml, + } + case *gitlab.TagEvent: + return &gitlabTagEventMatcher{ + workflow: workflow, + log: log, + event: evt, + trigger: trigger, + isYaml: isyaml, } } @@ -116,65 +177,335 @@ func createGitlabEventMatcher( } type gitlabPushEventMatcher struct { - log *zap.SugaredLogger - workflow *commonmodels.Workflow - event *gitlab.PushEvent + log *zap.SugaredLogger + workflow *commonmodels.Workflow + event *gitlab.PushEvent + trigger *TriggerYaml + isYaml bool + yamlServiceChanged []BuildServices } -func (gpem *gitlabPushEventMatcher) Match(hookRepo commonmodels.MainHookRepo) (bool, error) { +func (gpem *gitlabPushEventMatcher) Match(hookRepo *commonmodels.MainHookRepo) (bool, error) { ev := gpem.event - if (hookRepo.RepoOwner + "/" + hookRepo.RepoName) == ev.Project.PathWithNamespace { - if hookRepo.Branch == getBranchFromRef(ev.Ref) && EventConfigured(hookRepo, config.HookEventPush) { - var changedFiles []string - - detail, err := codehost.GetCodehostDetail(hookRepo.CodehostID) - if err != nil { - gpem.log.Errorf("GetCodehostDetail error: %v", err) - return false, err + if !checkRepoNamespaceMatch(hookRepo, ev.Project.PathWithNamespace) { + return false, nil + } + if !EventConfigured(hookRepo, config.HookEventPush) { + return false, nil + } + if gpem.isYaml { + refFlag := false + for _, ref := range gpem.trigger.Rules.Branchs { + if matched, _ := regexp.MatchString(ref, getBranchFromRef(ev.Ref)); matched { + refFlag = true + break } - - client, err := gitlabtool.NewClient(detail.Address, detail.OauthToken) - if err != nil { - gpem.log.Errorf("NewClient error: %v", err) - return false, err + } + if !refFlag { + return false, nil + } + } else { + isRegular := hookRepo.IsRegular + if !isRegular && hookRepo.Branch != getBranchFromRef(ev.Ref) { + return false, nil + } + if isRegular { + if matched, _ := regexp.MatchString(hookRepo.Branch, getBranchFromRef(ev.Ref)); !matched { + return false, nil } + } + } - // compare接口获取两个commit之间的最终的改动 - diffs, err := client.Compare(ev.ProjectID, ev.Before, ev.After) - if err != nil { - gpem.log.Errorf("Failed to get push event diffs, error: %v", err) - return false, err - } - for _, diff := range diffs { - changedFiles = append(changedFiles, diff.NewPath) - changedFiles = append(changedFiles, diff.OldPath) - } + hookRepo.Branch = getBranchFromRef(ev.Ref) + hookRepo.Committer = ev.UserUsername + var changedFiles []string + detail, err := systemconfig.New().GetCodeHost(hookRepo.CodehostID) + if err != nil { + gpem.log.Errorf("GetCodehostDetail error: %s", err) + return false, err + } - return MatchChanges(hookRepo, changedFiles), nil - } + client, err := gitlabtool.NewClient(detail.ID, detail.Address, detail.AccessToken, config.ProxyHTTPSAddr(), detail.EnableProxy) + if err != nil { + gpem.log.Errorf("NewClient error: %s", err) + return false, err } - return false, nil + // compare接口获取两个commit之间的最终的改动 + diffs, err := client.Compare(ev.ProjectID, ev.Before, ev.After) + if err != nil { + gpem.log.Errorf("Failed to get push event diffs, error: %s", err) + return false, err + } + for _, diff := range diffs { + changedFiles = append(changedFiles, diff.NewPath) + changedFiles = append(changedFiles, diff.OldPath) + } + if gpem.isYaml { + serviceChangeds := ServicesMatchChangesFiles(gpem.trigger.Rules.MatchFolders, changedFiles) + gpem.yamlServiceChanged = serviceChangeds + return len(serviceChangeds) != 0, nil + } + return MatchChanges(hookRepo, changedFiles), nil } func (gpem *gitlabPushEventMatcher) UpdateTaskArgs( - product *commonmodels.Product, args *commonmodels.WorkflowTaskArgs, hookRepo commonmodels.MainHookRepo, requestID string, + product *commonmodels.Product, args *commonmodels.WorkflowTaskArgs, hookRepo *commonmodels.MainHookRepo, requestID string, ) *commonmodels.WorkflowTaskArgs { + + if gpem.isYaml { + var targets []*commonmodels.TargetArgs + for _, target := range args.Target { + for _, bs := range gpem.yamlServiceChanged { + if target.Name == bs.ServiceModule && target.ServiceName == bs.Name { + targets = append(targets, target) + } + } + } + args.Target = targets + } factory := &workflowArgsFactory{ workflow: gpem.workflow, reqID: requestID, + IsYaml: gpem.isYaml, + } + + factory.Update(product, args, &types.Repository{ + CodehostID: hookRepo.CodehostID, + RepoName: hookRepo.RepoName, + RepoOwner: hookRepo.RepoOwner, + RepoNamespace: hookRepo.GetRepoNamespace(), + Branch: hookRepo.Branch, + }) + + return args +} + +type gitlabTagEventMatcher struct { + log *zap.SugaredLogger + workflow *commonmodels.Workflow + event *gitlab.TagEvent + trigger *TriggerYaml + isYaml bool + yamlServiceChanged []BuildServices +} + +func (gtem gitlabTagEventMatcher) Match(hookRepo *commonmodels.MainHookRepo) (bool, error) { + ev := gtem.event + + if !checkRepoNamespaceMatch(hookRepo, ev.Project.PathWithNamespace) { + return false, nil + } + + if !EventConfigured(hookRepo, config.HookEventTag) { + return false, nil + } + if gtem.isYaml { + refFlag := false + for _, ref := range gtem.trigger.Rules.Branchs { + if matched, _ := regexp.MatchString(ref, ev.Project.DefaultBranch); matched { + refFlag = true + break + } + } + if !refFlag { + return false, nil + } + } else { + isRegular := hookRepo.IsRegular + if !isRegular && hookRepo.Branch != ev.Project.DefaultBranch { + return false, nil + } + if isRegular { + if matched, _ := regexp.MatchString(hookRepo.Branch, ev.Project.DefaultBranch); !matched { + return false, nil + } + } + } + + hookRepo.Committer = ev.UserName + hookRepo.Tag = getTagFromRef(ev.Ref) + + return true, nil +} + +func (gtem gitlabTagEventMatcher) UpdateTaskArgs(product *commonmodels.Product, args *commonmodels.WorkflowTaskArgs, hookRepo *commonmodels.MainHookRepo, requestID string) *commonmodels.WorkflowTaskArgs { + if gtem.isYaml { + var targets []*commonmodels.TargetArgs + for _, target := range args.Target { + targets = append(targets, target) + } + args.Target = targets + } + factory := &workflowArgsFactory{ + workflow: gtem.workflow, + reqID: requestID, + IsYaml: gtem.isYaml, } factory.Update(product, args, &types.Repository{ - CodehostID: hookRepo.CodehostID, - RepoName: hookRepo.RepoName, - RepoOwner: hookRepo.RepoOwner, - Branch: hookRepo.Branch, + CodehostID: hookRepo.CodehostID, + RepoName: hookRepo.RepoName, + RepoOwner: hookRepo.RepoOwner, + RepoNamespace: hookRepo.GetRepoNamespace(), + Branch: hookRepo.Branch, + Tag: hookRepo.Tag, }) return args } +func UpdateWorkflowTaskArgs(triggerYaml *TriggerYaml, workflow *commonmodels.Workflow, workFlowArgs *commonmodels.WorkflowTaskArgs, item *commonmodels.WorkflowHook, branref string, prId int) error { + svcType, err := getServiceTypeByProject(workflow.ProductTmplName) + if err != nil { + return fmt.Errorf("getServiceTypeByProduct ProductTmplName:%s err:%s", workflow.ProductTmplName, err) + } + + ch, err := systemconfig.New().GetCodeHost(item.MainRepo.CodehostID) + if err != nil { + return fmt.Errorf("GetCodeHost codehostId:%d err:%s", item.MainRepo.CodehostID, err) + } + cli, err := gitlabtool.NewClient(ch.ID, ch.Address, ch.AccessToken, config.ProxyHTTPSAddr(), ch.EnableProxy) + if err != nil { + return fmt.Errorf("gitlabtool.NewClient codehostId:%d err:%s", item.MainRepo.CodehostID, err) + } + zadigTriggerYamls, err := cli.GetYAMLContents(item.MainRepo.RepoOwner, item.MainRepo.RepoName, item.YamlPath, branref, false, false) + if err != nil { + return fmt.Errorf("GetYAMLContents repoowner:%s reponame:%s ref:%s triggeryaml:%s err:%s", item.MainRepo.RepoOwner, item.MainRepo.RepoName, item.YamlPath, branref, err) + } + if len(zadigTriggerYamls) == 0 { + return fmt.Errorf("GetYAMLContents repoowner:%s reponame:%s ref:%s triggeryaml:%s ;content is empty", item.MainRepo.RepoOwner, item.MainRepo.RepoName, item.YamlPath, branref) + } + log.Infof("zadig-Trigger Yaml info:%s", zadigTriggerYamls[0]) + err = yaml.Unmarshal([]byte(zadigTriggerYamls[0]), triggerYaml) + if err != nil { + return fmt.Errorf("yaml.Unmarshal err:%s", err) + } + triggerYamlByt, err := json.Marshal(triggerYaml) + if err != nil { + return err + } + log.Infof("triggerYaml struct info:%s", string(triggerYamlByt)) + err = checkTriggerYamlParams(triggerYaml) + if err != nil { + return fmt.Errorf("checkTriggerYamlParams yamlPath:%s err:%s", item.YamlPath, err) + } + deployed := existStage(StageDeploy, triggerYaml) + if svcType == setting.BasicFacilityCVM { + deployed = true + } + workFlowArgs.WorkflowName = workflow.Name + workFlowArgs.ProductTmplName = workflow.ProductTmplName + if triggerYaml.CacheSet != nil { + workFlowArgs.IgnoreCache = triggerYaml.CacheSet.IgnoreCache + workFlowArgs.ResetCache = triggerYaml.CacheSet.ResetCache + } + if triggerYaml.Deploy != nil { + workFlowArgs.Namespace = strings.Join(triggerYaml.Deploy.Envsname, ",") + workFlowArgs.BaseNamespace = triggerYaml.Deploy.BaseNamespace + workFlowArgs.EnvRecyclePolicy = string(triggerYaml.Deploy.EnvRecyclePolicy) + workFlowArgs.EnvUpdatePolicy = string(triggerYaml.Deploy.Strategy) + } + item.MainRepo.Events = triggerYaml.Rules.Events + if triggerYaml.Rules.Strategy != nil { + item.AutoCancel = triggerYaml.Rules.Strategy.AutoCancel + } + //test + tests := make([]*commonmodels.TestArgs, 0) + for _, test := range triggerYaml.Test { + moduleTest, err := commonrepo.NewTestingColl().Find(test.Name, workflow.ProductTmplName) + if err != nil { + log.Errorf("fail to find test TestModuleName:%s, workflowname:%s,productTmplName:%s,error:%v", test.Name, workflow.Name, workflow.ProductTmplName, err) + if commonrepo.IsErrNoDocuments(err) { + continue + } + return fmt.Errorf("fail to find test TestModuleName:%s, workflowname:%s,productTmplName:%s,error:%s", test.Name, workflow.Name, workflow.ProductTmplName, err) + } + envs := make([]*commonmodels.KeyVal, 0) + for _, env := range test.Variables { + envElem := &commonmodels.KeyVal{ + Key: env.Name, + Value: env.Value, + } + envs = append(envs, envElem) + } + testArg := &commonmodels.TestArgs{ + TestModuleName: test.Name, + Envs: envs, + } + if test.Repo.Strategy == TestRepoStrategyCurrentRepo { + for _, repo := range moduleTest.Repos { + if repo.RepoName == item.MainRepo.RepoName && repo.RepoOwner == item.MainRepo.RepoOwner { + repo.Branch = branref + repo.PR = prId + } + } + } + testArg.Builds = moduleTest.Repos + tests = append(tests, testArg) + } + workFlowArgs.Tests = tests + testsRepo, err := json.Marshal(workFlowArgs.Tests) + if err != nil { + log.Errorf("json.Marshal workflowname:%s,productTmplName:%s,error:%s", workflow.Name, workflow.ProductTmplName, err) + return fmt.Errorf("json.Marshal workflowname:%s,productTmplName:%s,error:%s", workflow.Name, workflow.ProductTmplName, err) + } + log.Infof("moduleTests workflowname:%s,productTmplName:%s,info:%s", workflow.Name, workflow.ProductTmplName, string(testsRepo)) + //target + targets := make([]*commonmodels.TargetArgs, 0) + for _, svr := range triggerYaml.Rules.MatchFolders.MatchFoldersTree { + targetElem := &commonmodels.TargetArgs{ + Name: svr.ServiceModule, + ProductName: workflow.ProductTmplName, + ServiceName: svr.Name, + ServiceType: svcType, + } + opt := &commonrepo.BuildFindOption{ + ServiceName: svr.Name, + ProductName: workflow.ProductTmplName, + Targets: []string{svr.ServiceModule}, + } + resp, err := commonrepo.NewBuildColl().Find(opt) + if err != nil { + log.Errorf("[Build.Find] serviceName: %s productName:%s serviceModule:%s error: %s", svr.Name, workflow.ProductTmplName, svr.ServiceModule, err) + if commonrepo.IsErrNoDocuments(err) { + continue + } + return fmt.Errorf("[Build.Find] serviceName: %s productName:%s serviceModule:%s error: %s", svr.Name, workflow.ProductTmplName, svr.ServiceModule, err) + } + + repos := commonservice.FindReposByTarget(targetElem.ProductName, targetElem.ServiceName, targetElem.Name, resp) + for _, repo := range repos { + if repo.RepoName == item.MainRepo.RepoName && repo.RepoOwner == item.MainRepo.RepoOwner { + repo.Branch = branref + repo.PR = prId + } + } + targetElem.Build = &commonmodels.BuildArgs{Repos: repos} + + targetElem.Deploy = make([]commonmodels.DeployEnv, 0) + if deployed { + targetElem.Deploy = append(targetElem.Deploy, commonmodels.DeployEnv{Env: svr.Name + "/" + svr.ServiceModule, Type: targetElem.ServiceType}) + } + var envs []*commonmodels.KeyVal + for _, bsvr := range triggerYaml.Build { + if bsvr.Name == svr.Name && bsvr.ServiceModule == svr.ServiceModule { + for _, env := range bsvr.Variables { + envElem := &commonmodels.KeyVal{ + Key: env.Name, + Value: env.Value, + } + envs = append(envs, envElem) + } + } + } + targetElem.Envs = envs + targets = append(targets, targetElem) + } + workFlowArgs.Target = targets + return nil +} + func TriggerWorkflowByGitlabEvent(event interface{}, baseURI, requestID string, log *zap.SugaredLogger) error { // TODO: cache workflow // 1. find configured workflow @@ -198,12 +529,58 @@ func TriggerWorkflowByGitlabEvent(event interface{}, baseURI, requestID string, log.Debugf("find %d hooks in workflow %s", len(workflow.HookCtl.Items), workflow.Name) for _, item := range workflow.HookCtl.Items { - if item.WorkflowArgs == nil { + if item.WorkflowArgs == nil && !item.IsYaml { continue } + triggerYaml := &TriggerYaml{} + workFlowArgs := &commonmodels.WorkflowTaskArgs{} + var pushEvent *gitlab.PushEvent + var mergeEvent *gitlab.MergeEvent + var tagEvent *gitlab.TagEvent + prID := 0 + branref := "" + switch evt := event.(type) { + case *gitlab.PushEvent: + pushEvent = evt + if !checkRepoNamespaceMatch(item.MainRepo, pushEvent.Project.PathWithNamespace) { + log.Debugf("event not matches repo: %v", item.MainRepo) + continue + } + branref = pushEvent.Ref + case *gitlab.MergeEvent: + mergeEvent = evt + if !checkRepoNamespaceMatch(item.MainRepo, mergeEvent.ObjectAttributes.Target.PathWithNamespace) { + log.Debugf("event not matches repo: %v", item.MainRepo) + continue + } + if mergeEvent.ObjectAttributes.Source.PathWithNamespace != mergeEvent.ObjectAttributes.Target.PathWithNamespace { + branref = mergeEvent.ObjectAttributes.TargetBranch + } else { + branref = mergeEvent.ObjectAttributes.SourceBranch + } + prID = evt.ObjectAttributes.IID + case *gitlab.TagEvent: + tagEvent = evt + if !checkRepoNamespaceMatch(item.MainRepo, tagEvent.Project.PathWithNamespace) { + log.Debugf("event not matches repo: %v", item.MainRepo) + continue + } + branref = tagEvent.Ref + } + if item.IsYaml { + err := UpdateWorkflowTaskArgs(triggerYaml, workflow, workFlowArgs, item, branref, prID) + if err != nil { + log.Warnf("UpdateWorkflowTaskArgs %s", err) + mErr = multierror.Append(mErr, err) + continue + } + item.WorkflowArgs = workFlowArgs + } else { + workFlowArgs = item.WorkflowArgs + } // 2. match webhook - matcher := createGitlabEventMatcher(event, diffSrv, workflow, log) + matcher := createGitlabEventMatcher(event, diffSrv, workflow, item.IsYaml, triggerYaml, log) if matcher == nil { continue } @@ -218,7 +595,6 @@ func TriggerWorkflowByGitlabEvent(event interface{}, baseURI, requestID string, log.Debugf("event not matches %v", item.MainRepo) continue } - log.Infof("event match hook %v of %s", item.MainRepo, workflow.Name) namespace := strings.Split(item.WorkflowArgs.Namespace, ",")[0] opt := &commonrepo.ProductFindOptions{Name: workflow.ProductTmplName, EnvName: namespace} @@ -229,11 +605,9 @@ func TriggerWorkflowByGitlabEvent(event interface{}, baseURI, requestID string, } isMergeRequest := false - prID := 0 var mergeRequestID, commitID string if ev, isPr := event.(*gitlab.MergeEvent); isPr { isMergeRequest = true - prID = ev.ObjectAttributes.IID // 如果是merge request，且该webhook触发器配置了自动取消， // 则需要确认该merge request在本次commit之前的commit触发的任务是否处理完，没有处理完则取消掉。 @@ -244,43 +618,44 @@ func TriggerWorkflowByGitlabEvent(event interface{}, baseURI, requestID string, CommitID: commitID, TaskType: config.WorkflowType, MainRepo: item.MainRepo, - WorkflowArgs: item.WorkflowArgs, + WorkflowArgs: workFlowArgs, + IsYaml: item.IsYaml, + AutoCancel: item.AutoCancel, + YamlHookPath: item.YamlPath, } err := AutoCancelTask(autoCancelOpt, log) if err != nil { log.Errorf("failed to auto cancel workflow task when receive event %v due to %v ", event, err) mErr = multierror.Append(mErr, err) } - if notification == nil { notification, _ = scmnotify.NewService().SendInitWebhookComment( - &item.MainRepo, ev.ObjectAttributes.IID, baseURI, false, false, log, + item.MainRepo, ev.ObjectAttributes.IID, baseURI, false, false, false, log, ) - - // 初始化 gitlab diff_note - InitDiffNote(ev, item.MainRepo, log) } } if notification != nil { - item.WorkflowArgs.NotificationID = notification.ID.Hex() + workFlowArgs.NotificationID = notification.ID.Hex() } - args := matcher.UpdateTaskArgs(prod, item.WorkflowArgs, item.MainRepo, requestID) + args := matcher.UpdateTaskArgs(prod, workFlowArgs, item.MainRepo, requestID) args.MergeRequestID = mergeRequestID args.CommitID = commitID args.Source = setting.SourceFromGitlab args.CodehostID = item.MainRepo.CodehostID args.RepoOwner = item.MainRepo.RepoOwner + args.RepoNamespace = item.MainRepo.GetRepoNamespace() args.RepoName = item.MainRepo.RepoName + args.Committer = item.MainRepo.Committer // 3. create task with args if item.WorkflowArgs.BaseNamespace == "" { - if resp, err := workflowservice.CreateWorkflowTask(args, setting.WebhookTaskCreator, permission.AnonymousUserID, false, log); err != nil { + if resp, err := workflowservice.CreateWorkflowTask(args, setting.WebhookTaskCreator, log); err != nil { log.Errorf("failed to create workflow task when receive push event %v due to %v ", event, err) mErr = multierror.Append(mErr, err) // 单独创建一条通知，展示任务创建失败的错误信息 _, err2 := scmnotify.NewService().SendErrWebhookComment( - &item.MainRepo, workflow, err, prID, baseURI, false, false, log, + item.MainRepo, workflow, err, prID, baseURI, false, false, log, ) if err2 != nil { log.Errorf("SendErrWebhookComment failed, product:%s, workflow:%s, err:%v", workflow.ProductTmplName, workflow.Name, err2) @@ -292,6 +667,8 @@ func TriggerWorkflowByGitlabEvent(event interface{}, baseURI, requestID string, if err = CreateEnvAndTaskByPR(args, prID, requestID, log); err != nil { log.Infof("CreateRandomEnv err:%v", err) } + } else { + log.Warnf("It's not a PR event,BaseNamespace:%s", item.WorkflowArgs.BaseNamespace) } } } @@ -300,12 +677,12 @@ func TriggerWorkflowByGitlabEvent(event interface{}, baseURI, requestID string, } func findChangedFilesOfMergeRequest(event *gitlab.MergeEvent, codehostID int) ([]string, error) { - detail, err := codehost.GetCodehostDetail(codehostID) + detail, err := systemconfig.New().GetCodeHost(codehostID) if err != nil { return nil, fmt.Errorf("failed to find codehost %d: %v", codehostID, err) } - client, err := gitlabtool.NewClient(detail.Address, detail.OauthToken) + client, err := gitlabtool.NewClient(detail.ID, detail.Address, detail.AccessToken, config.ProxyHTTPSAddr(), detail.EnableProxy) if err != nil { log.Error(err) return nil, e.ErrCodehostListProjects.AddDesc(err.Error()) @@ -314,101 +691,6 @@ func findChangedFilesOfMergeRequest(event *gitlab.MergeEvent, codehostID int) ([ return client.ListChangedFiles(event) } -// InitDiffNote 调用gitlab接口初始化DiffNote，并保存到数据库 -func InitDiffNote(ev *gitlab.MergeEvent, mainRepo commonmodels.MainHookRepo, log *zap.SugaredLogger) error { - commitID := ev.ObjectAttributes.LastCommit.ID - body := "KodeRover CI 检查中..." - - // 调用gitlab api获取相关数据 - detail, err := codehost.GetCodehostDetail(mainRepo.CodehostID) - if err != nil { - log.Errorf("GetCodehostDetail failed, codehost:%d, err:%v", mainRepo.CodehostID, err) - return fmt.Errorf("failed to find codehost %d: %v", mainRepo.CodehostID, err) - } - cli, _ := gitlab.NewOAuthClient(detail.OauthToken, gitlab.WithBaseURL(detail.Address)) - - opt := &commonrepo.DiffNoteFindOpt{ - CodehostID: mainRepo.CodehostID, - ProjectID: mainRepo.RepoOwner + "/" + mainRepo.RepoName, - MergeRequestID: ev.ObjectAttributes.IID, - } - dn, err := commonrepo.NewDiffNoteColl().Find(opt) - if err == nil { - // 该pr的DiffNote已经创建过，且是同一个commit，则不处理 - if dn.CommitID == commitID { - return nil - } - // 不是同一个commit，则重置body和resolved - // 更新note body - noteBodyOpt := &gitlab.UpdateMergeRequestDiscussionNoteOptions{ - Body: &body, - } - _, _, err = cli.Discussions.UpdateMergeRequestDiscussionNote(dn.Repo.ProjectID, dn.MergeRequestID, dn.DiscussionID, dn.NoteID, noteBodyOpt) - if err != nil { - log.Errorf("UpdateMergeRequestDiscussionNote failed, err:%v", err) - return err - } - - // 更新resolved状态 - resolved := false - resolveOpt := &gitlab.UpdateMergeRequestDiscussionNoteOptions{ - Resolved: &resolved, - } - _, _, err = cli.Discussions.UpdateMergeRequestDiscussionNote(dn.Repo.ProjectID, dn.MergeRequestID, dn.DiscussionID, dn.NoteID, resolveOpt) - if err != nil { - log.Errorf("UpdateMergeRequestDiscussionNote failed, err:%v", err) - return err - } - - // 更新到数据库 - dn.Resolved = resolved - dn.Body = body - err = commonrepo.NewDiffNoteColl().Update(dn.ObjectID.Hex(), commitID, dn.Body, dn.Resolved) - if err != nil { - log.Errorf("UpdateDiscussionInfo failed, err:%v", err) - return err - } - - return nil - } - - // 不存在则创建 - diffNote := &commonmodels.DiffNote{ - Repo: &commonmodels.RepoInfo{ - CodehostID: mainRepo.CodehostID, - Source: "gitlab", - ProjectID: mainRepo.RepoOwner + "/" + mainRepo.RepoName, - Address: detail.Address, - OauthToken: detail.OauthToken, - }, - MergeRequestID: ev.ObjectAttributes.IID, - CommitID: commitID, - Body: body, - } - - createOpt := &gitlab.CreateMergeRequestDiscussionOptions{ - Body: &diffNote.Body, - } - - discussion, _, err := cli.Discussions.CreateMergeRequestDiscussion(diffNote.Repo.ProjectID, diffNote.MergeRequestID, createOpt) - if err != nil { - log.Errorf("CreateMergeRequestDiscussion failed, err:%v", err) - return err - } - - diffNote.DiscussionID = discussion.ID - if len(discussion.Notes) > 0 { - diffNote.NoteID = discussion.Notes[0].ID - } - err = commonrepo.NewDiffNoteColl().Create(diffNote) - if err != nil { - log.Errorf("DiffNote.Create failed, err:%v", err) - return err - } - - return nil -} - var mutex sync.Mutex // CreateEnvAndTaskByPR 根据pr触发创建环境、使用工作流更新该创建的环境、根据环境删除策略删除环境 @@ -425,7 +707,7 @@ func CreateEnvAndTaskByPR(workflowArgs *commonmodels.WorkflowTaskArgs, prID int, mutex.Unlock() }() if baseProduct.Render != nil { - if renderSet, _ := commonrepo.NewRenderSetColl().Find(&commonrepo.RenderSetFindOption{Name: baseProduct.Render.Name, Revision: baseProduct.Render.Revision}); renderSet != nil { + if renderSet, _ := commonrepo.NewRenderSetColl().Find(&commonrepo.RenderSetFindOption{Name: baseProduct.Render.Name, Revision: baseProduct.Render.Revision, ProductTmpl: baseProduct.ProductName}); renderSet != nil { baseProduct.Vars = renderSet.KVs } } @@ -447,7 +729,7 @@ func CreateEnvAndTaskByPR(workflowArgs *commonmodels.WorkflowTaskArgs, prID int, } workflowArgs.Namespace = envName - taskResp, err := workflowservice.CreateWorkflowTask(workflowArgs, setting.WebhookTaskCreator, permission.AnonymousUserID, false, log) + taskResp, err := workflowservice.CreateWorkflowTask(workflowArgs, setting.WebhookTaskCreator, log) if err != nil { return fmt.Errorf("CreateEnvAndTaskByPR CreateWorkflowTask err：%v ", err) } @@ -470,7 +752,7 @@ func CreateEnvAndTaskByPR(workflowArgs *commonmodels.WorkflowTaskArgs, prID int, } //按照用户设置的环境回收策略进行环境回收 if workflowArgs.EnvRecyclePolicy == setting.EnvRecyclePolicyAlways || (workflowArgs.EnvRecyclePolicy == setting.EnvRecyclePolicyTaskStatus && taskStatus == string(config.StatusPassed)) { - err = commonservice.DeleteProduct(setting.SystemUser, envName, workflowArgs.ProductTmplName, requestID, log) + err = environmentservice.DeleteProduct(setting.SystemUser, envName, workflowArgs.ProductTmplName, requestID, true, log) if err != nil { log.Errorf("CreateEnvAndTaskByPR DeleteProduct err:%v ", err) return err diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/ilyshin.go b/pkg/microservice/aslan/core/workflow/service/webhook/ilyshin.go deleted file mode 100644 index 491d15142f482fed32fdb8bc30f83ba1e9e88d02..0000000000000000000000000000000000000000 --- a/pkg/microservice/aslan/core/workflow/service/webhook/ilyshin.go +++ /dev/null @@ -1,291 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package webhook - -import ( - "errors" - "fmt" - "net/http" - "strconv" - "strings" - "time" - - "github.com/hashicorp/go-multierror" - "go.uber.org/zap" - - "github.com/koderover/zadig/pkg/microservice/aslan/config" - commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" - commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" - gitservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/git" - workflowservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" - "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/codehost" - e "github.com/koderover/zadig/pkg/tool/errors" - "github.com/koderover/zadig/pkg/tool/ilyshin" - "github.com/koderover/zadig/pkg/types/permission" -) - -func ProcessIlyshinHook(payload []byte, req *http.Request, requestID string, log *zap.SugaredLogger) error { - token := req.Header.Get("X-CodeHub-Token") - secret := gitservice.GetHookSecret() - - if secret != "" && token != secret { - return errors.New("token is illegal") - } - - eventType := ilyshin.HookEventType(req) - event, err := ilyshin.ParseHook(eventType, payload) - if err != nil { - log.Warnf("unexpected event type: %s", eventType) - return nil - } - - forwardedProto := req.Header.Get("X-Forwarded-Proto") - forwardedHost := req.Header.Get("X-Forwarded-Host") - baseURI := fmt.Sprintf("%s://%s", forwardedProto, forwardedHost) - - var pushEvent *ilyshin.PushEvent - var errorList = &multierror.Error{} - switch event := event.(type) { - case *ilyshin.PushEvent: - pushEvent = event - if len(pushEvent.Commits) > 0 { - webhookUser := &commonmodels.WebHookUser{ - Domain: req.Header.Get("X-Forwarded-Host"), - UserName: pushEvent.Commits[0].Author.Name, - Email: pushEvent.Commits[0].Author.Email, - Source: setting.SourceFromIlyshin, - CreatedAt: time.Now().Unix(), - } - commonrepo.NewWebHookUserColl().Upsert(webhookUser) - } - - if err = updateServiceTemplateByIlyshinPushEvent(pushEvent, log); err != nil { - errorList = multierror.Append(errorList, err) - } - } - - //产品工作流webhook - if err = TriggerWorkflowByIlyshinEvent(event, baseURI, requestID, log); err != nil { - errorList = multierror.Append(errorList, err) - } - - return errorList.ErrorOrNil() -} - -func updateServiceTemplateByIlyshinPushEvent(event *ilyshin.PushEvent, log *zap.SugaredLogger) error { - log.Infof("EVENT: Ilyshin WEBHOOK UPDATING SERVICE TEMPLATE") - - address, err := GetAddress(event.Project.WebURL) - if err != nil { - log.Errorf("GetAddress failed, error: %s", err) - return err - } - - client, err := getIlyshinClientByAddress(address) - if err != nil { - return err - } - - diffs, err := client.Compare(event.ProjectID, event.Before, event.After) - if err != nil { - log.Errorf("Failed to get push event diffs, error: %s", err) - return err - } - serviceTmpls, err := GetIlyshinServiceTemplates() - if err != nil { - log.Errorf("Failed to get ilyshin service templates, error: %s", err) - return err - } - - errs := &multierror.Error{} - for _, service := range serviceTmpls { - srcPath := service.SrcPath - _, _, _, _, path, _, err := GetOwnerRepoBranchPath(srcPath) - if err != nil { - errs = multierror.Append(errs, err) - } - // 判断PushEvent的Diffs中是否包含该服务模板的src_path - affected := false - for _, diff := range diffs { - if strings.Contains(diff.OldPath, path) || strings.Contains(diff.NewPath, path) { - affected = true - break - } - } - if affected { - log.Infof("Started to sync service template %s from ilyshin %s", service.ServiceName, service.SrcPath) - service.CreateBy = "system" - err := SyncServiceTemplateFromIlyshin(service, log) - if err != nil { - log.Errorf("SyncServiceTemplateFromIlyshin failed, error: %s", err) - errs = multierror.Append(errs, err) - } - } else { - log.Infof("Service template %s from ilyshin %s is not affected, no sync", service.ServiceName, service.SrcPath) - } - - } - return errs.ErrorOrNil() -} - -func GetIlyshinServiceTemplates() ([]*commonmodels.Service, error) { - opt := &commonrepo.ServiceListOption{ - Type: setting.K8SDeployType, - Source: setting.SourceFromIlyshin, - } - return commonrepo.NewServiceColl().ListMaxRevisions(opt) -} - -// SyncServiceTemplateFromIlyshin Force to sync Service Template to latest commit and content, -// Notes: if remains the same, quit sync; if updates, revision +1 -func SyncServiceTemplateFromIlyshin(service *commonmodels.Service, log *zap.SugaredLogger) error { - // 判断一下 Source 字段，如果 Source 字段不是 ilyshin，直接返回 - if service.Source != setting.SourceFromIlyshin { - return fmt.Errorf("service template is not from ilyshin") - } - // 获取当前Commit的SHA - var before string - if service.Commit != nil { - before = service.Commit.SHA - } - // Sync最新的Commit的SHA - var after string - err := syncIlyshinLatestCommit(service, log) - if err != nil { - return err - } - after = service.Commit.SHA - // 判断一下是否需要 Sync 内容 - if before == after { - log.Infof("Before and after SHA: %s remains the same, no need to sync", before) - // 无需更新 - return nil - } - // 在 Ensure 过程中会检查 source，如果 source 为 ilyshin，则同步 ilyshin 内容到 service 中 - if err := fillServiceTmpl(setting.WebhookTaskCreator, service, log); err != nil { - log.Errorf("ensureServiceTmpl error: %+v", err) - return e.ErrValidateTemplate.AddDesc(err.Error()) - } - // 更新到数据库，revision+1 - if err := commonrepo.NewServiceColl().Create(service); err != nil { - log.Errorf("Failed to sync service %s from ilyshin path %s error: %v", service.ServiceName, service.SrcPath, err) - return e.ErrCreateTemplate.AddDesc(err.Error()) - } - log.Infof("End of sync service template %s from ilyshin path %s", service.ServiceName, service.SrcPath) - return nil -} - -func TriggerWorkflowByIlyshinEvent(event interface{}, baseURI, requestID string, log *zap.SugaredLogger) error { - // 1. find configured workflow - workflowList, err := commonrepo.NewWorkflowColl().List(&commonrepo.ListWorkflowOption{}) - if err != nil { - log.Errorf("failed to list workflow %v", err) - return err - } - - mErr := &multierror.Error{} - diffSrv := func(mergeEvent *ilyshin.MergeEvent, codehostId int) ([]string, error) { - return findIlyshinChangedFilesOfMergeRequest(mergeEvent, codehostId) - } - - for _, workflow := range workflowList { - if workflow.HookCtl == nil || !workflow.HookCtl.Enabled { - continue - } - - log.Debugf("find %d hooks in workflow %s", len(workflow.HookCtl.Items), workflow.Name) - for _, item := range workflow.HookCtl.Items { - if item.WorkflowArgs == nil { - continue - } - - // 2. match webhook - matcher := createIlyshinEventMatcher(event, diffSrv, workflow, log) - if matcher == nil { - continue - } - - matches, err := matcher.Match(item.MainRepo) - if err != nil { - mErr = multierror.Append(mErr, err) - continue - } - - if !matches { - log.Debugf("event not matches %v", item.MainRepo) - continue - } - - log.Infof("event match hook %v of %s", item.MainRepo, workflow.Name) - namespace := strings.Split(item.WorkflowArgs.Namespace, ",")[0] - opt := &commonrepo.ProductFindOptions{Name: workflow.ProductTmplName, EnvName: namespace} - var prod *commonmodels.Product - if prod, err = commonrepo.NewProductColl().Find(opt); err != nil { - log.Warnf("can't find environment %s-%s", item.WorkflowArgs.Namespace, workflow.ProductTmplName) - continue - } - - var mergeRequestID, commitID string - if ev, isPr := event.(*ilyshin.MergeEvent); isPr { - // 如果是merge request，且该webhook触发器配置了自动取消， - // 则需要确认该merge request在本次commit之前的commit触发的任务是否处理完，没有处理完则取消掉。 - mergeRequestID = strconv.Itoa(ev.ObjectAttributes.IID) - commitID = ev.ObjectAttributes.LastCommit.ID - autoCancelOpt := &AutoCancelOpt{ - MergeRequestID: mergeRequestID, - CommitID: commitID, - TaskType: config.WorkflowType, - MainRepo: item.MainRepo, - WorkflowArgs: item.WorkflowArgs, - } - err := AutoCancelTask(autoCancelOpt, log) - if err != nil { - log.Errorf("failed to auto cancel workflow task when receive event %v due to %v ", event, err) - mErr = multierror.Append(mErr, err) - } - } - - args := matcher.UpdateTaskArgs(prod, item.WorkflowArgs, item.MainRepo, requestID) - args.MergeRequestID = mergeRequestID - args.CommitID = commitID - args.Source = setting.SourceFromIlyshin - args.CodehostID = item.MainRepo.CodehostID - args.RepoOwner = item.MainRepo.RepoOwner - args.RepoName = item.MainRepo.RepoName - // 3. create task with args - if _, err := workflowservice.CreateWorkflowTask(args, setting.WebhookTaskCreator, permission.AnonymousUserID, false, log); err != nil { - log.Errorf("failed to create workflow task when receive push event %v due to %s ", event, err) - mErr = multierror.Append(mErr, err) - } else { - log.Infof("succeed to create task") - } - } - } - - return mErr.ErrorOrNil() -} - -func findIlyshinChangedFilesOfMergeRequest(event *ilyshin.MergeEvent, codehostID int) ([]string, error) { - detail, err := codehost.GetCodehostDetail(codehostID) - if err != nil { - return nil, fmt.Errorf("failed to find codehost %d: %s", codehostID, err) - } - - client := ilyshin.NewClient(detail.Address, detail.OauthToken) - return client.ListChangedFiles(event) -} diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/ilyshin_workflow_task.go b/pkg/microservice/aslan/core/workflow/service/webhook/ilyshin_workflow_task.go deleted file mode 100644 index bea8048fc5fac36076ecd10a24265d1c7c152d3e..0000000000000000000000000000000000000000 --- a/pkg/microservice/aslan/core/workflow/service/webhook/ilyshin_workflow_task.go +++ /dev/null @@ -1,152 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package webhook - -import ( - "go.uber.org/zap" - - "github.com/koderover/zadig/pkg/microservice/aslan/config" - commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" - "github.com/koderover/zadig/pkg/shared/codehost" - "github.com/koderover/zadig/pkg/tool/ilyshin" - "github.com/koderover/zadig/pkg/types" -) - -type ilyshinMergeRequestDiffFunc func(event *ilyshin.MergeEvent, id int) ([]string, error) - -type ilyshinMergeEventMatcher struct { - diffFunc ilyshinMergeRequestDiffFunc - log *zap.SugaredLogger - workflow *commonmodels.Workflow - event *ilyshin.MergeEvent -} - -func (gmem *ilyshinMergeEventMatcher) Match(hookRepo commonmodels.MainHookRepo) (bool, error) { - ev := gmem.event - if (hookRepo.RepoOwner + "/" + hookRepo.RepoName) == ev.ObjectAttributes.Target.PathWithNamespace { - if EventConfigured(hookRepo, config.HookEventPr) && (hookRepo.Branch == ev.ObjectAttributes.TargetBranch) { - if ev.ObjectAttributes.State == "opened" { - var changedFiles []string - changedFiles, err := gmem.diffFunc(ev, hookRepo.CodehostID) - if err != nil { - gmem.log.Warnf("failed to get changes of event %v, err:%v", ev, err) - return false, err - } - gmem.log.Debugf("succeed to get %d changes in merge event", len(changedFiles)) - - return MatchChanges(hookRepo, changedFiles), nil - } - } - } - return false, nil -} - -func (gmem *ilyshinMergeEventMatcher) UpdateTaskArgs( - product *commonmodels.Product, args *commonmodels.WorkflowTaskArgs, hookRepo commonmodels.MainHookRepo, requestID string, -) *commonmodels.WorkflowTaskArgs { - factory := &workflowArgsFactory{ - workflow: gmem.workflow, - reqID: requestID, - } - - args = factory.Update(product, args, &types.Repository{ - CodehostID: hookRepo.CodehostID, - RepoName: hookRepo.RepoName, - RepoOwner: hookRepo.RepoOwner, - Branch: hookRepo.Branch, - PR: gmem.event.ObjectAttributes.IID, - }) - - return args -} - -func createIlyshinEventMatcher( - event interface{}, diffSrv ilyshinMergeRequestDiffFunc, workflow *commonmodels.Workflow, log *zap.SugaredLogger, -) gitEventMatcher { - switch evt := event.(type) { - case *ilyshin.PushEvent: - return &ilyshinPushEventMatcher{ - workflow: workflow, - log: log, - event: evt, - } - case *ilyshin.MergeEvent: - return &ilyshinMergeEventMatcher{ - diffFunc: diffSrv, - log: log, - event: evt, - workflow: workflow, - } - } - - return nil -} - -type ilyshinPushEventMatcher struct { - log *zap.SugaredLogger - workflow *commonmodels.Workflow - event *ilyshin.PushEvent -} - -func (gpem *ilyshinPushEventMatcher) Match(hookRepo commonmodels.MainHookRepo) (bool, error) { - ev := gpem.event - if (hookRepo.RepoOwner + "/" + hookRepo.RepoName) == ev.Project.PathWithNamespace { - if hookRepo.Branch == getBranchFromRef(ev.Ref) && EventConfigured(hookRepo, config.HookEventPush) { - var changedFiles []string - - detail, err := codehost.GetCodehostDetail(hookRepo.CodehostID) - if err != nil { - gpem.log.Errorf("GetCodehostDetail error: %v", err) - return false, err - } - - client := ilyshin.NewClient(detail.Address, detail.OauthToken) - // compare接口获取两个commit之间的最终的改动 - diffs, err := client.Compare(ev.ProjectID, ev.Before, ev.After) - if err != nil { - gpem.log.Errorf("Failed to get push event diffs, error: %v", err) - return false, err - } - for _, diff := range diffs { - changedFiles = append(changedFiles, diff.NewPath) - changedFiles = append(changedFiles, diff.OldPath) - } - - return MatchChanges(hookRepo, changedFiles), nil - } - } - - return false, nil -} - -func (gpem *ilyshinPushEventMatcher) UpdateTaskArgs( - product *commonmodels.Product, args *commonmodels.WorkflowTaskArgs, hookRepo commonmodels.MainHookRepo, requestID string, -) *commonmodels.WorkflowTaskArgs { - factory := &workflowArgsFactory{ - workflow: gpem.workflow, - reqID: requestID, - } - - factory.Update(product, args, &types.Repository{ - CodehostID: hookRepo.CodehostID, - RepoName: hookRepo.RepoName, - RepoOwner: hookRepo.RepoOwner, - Branch: hookRepo.Branch, - }) - - return args -} diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/trigger_yaml.go b/pkg/microservice/aslan/core/workflow/service/webhook/trigger_yaml.go new file mode 100644 index 0000000000000000000000000000000000000000..4728427401c5a17496a512c4bbb17692ea1bb3a1 --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/service/webhook/trigger_yaml.go @@ -0,0 +1,138 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "github.com/koderover/zadig/pkg/microservice/aslan/config" +) + +type Stage string + +const ( + StageBuild Stage = "build" + StageDeploy Stage = "deploy" + StageTest Stage = "test" +) + +type DeployStrategy string + +const ( + DeployStrategySingle DeployStrategy = "single" + DeployStrategyDynamic DeployStrategy = "dynamic" + DeployStrategyBase DeployStrategy = "base" +) + +type EnvRecyclePolicy string + +const ( + EnvRecyclePolicySuccess EnvRecyclePolicy = "success" + EnvRecyclePolicyAlways EnvRecyclePolicy = "always" + EnvRecyclePolicyNever EnvRecyclePolicy = "never" +) + +type TestRepoStrategy string + +const ( + TestRepoStrategyDefault TestRepoStrategy = "default" + TestRepoStrategyCurrentRepo TestRepoStrategy = "currentRepo" +) + +type TriggerYaml struct { + Stages []Stage `yaml:"stages"` + Build []*BuildServices `yaml:"build"` + Deploy *Deploy `yaml:"deploy"` + Test []*Test `yaml:"test"` + Rules *Rules `yaml:"rules"` + CacheSet *CacheSet `yaml:"cache_set"` +} + +type Build struct { + Services []*BuildServices `yaml:"services"` +} + +type BuildServices struct { + Name string `yaml:"name"` + ServiceModule string `yaml:"service_module"` + Variables []*Variables `yaml:"variables"` +} + +type Variables struct { + Name string `yaml:"name"` + Value string `yaml:"value"` +} + +/* +Strategy + single: + Update to the specified environment; + dynamic: + Dynamic selection of idle environment; + base: + Create a new environment based on the baseline environment + And Need to set EnvRecyclePolicy：success/always/never +*/ +type Deploy struct { + Strategy DeployStrategy `yaml:"strategy"` + Envsname []string `yaml:"envs_name"` + EnvRecyclePolicy EnvRecyclePolicy `yaml:"env_recycle_policy"` + BaseNamespace string `yaml:"base_env"` +} + +type Test struct { + Name string `yaml:"name"` + Repo *Repo `yaml:"repo"` + Variables []*Variables `yaml:"variables"` +} + +/* +Strategy + default： + Code repository information configured by default on the zadig platform + currentRepo： + Use current changed code information +*/ +type Repo struct { + Strategy TestRepoStrategy `yaml:"strategy"` +} + +type Rules struct { + Branchs []string `yaml:"branchs"` + Events []config.HookEventType `yaml:"events"` + Strategy *StrategyRules `yaml:"strategy"` + MatchFolders *MatchFoldersElem `yaml:"match_folders"` +} + +type StrategyRules struct { + AutoCancel bool `yaml:"auto_cancel"` +} + +type MatchFoldersElem struct { + MatchSwitch string `yaml:"match_switch"` + MatchFoldersTree []*MatchFoldersTree `yaml:"match_folders_tree"` + MatchFoldersSpecific string `yaml:"match_folders_specific"` +} + +type MatchFoldersTree struct { + Name string `yaml:"name"` + ServiceModule string `yaml:"service_module"` + FileTree []string `yaml:"file_tree"` +} + +type CacheSet struct { + IgnoreCache bool `yaml:"ignore_cache"` + ResetCache bool `yaml:"reset_cache"` +} diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/utils.go b/pkg/microservice/aslan/core/workflow/service/webhook/utils.go index 98fba1549676a0d8e4bf58bae770f42525936e47..90d51a48f785ddaf9ade39be43e80a0a22d27cf2 100644 --- a/pkg/microservice/aslan/core/workflow/service/webhook/utils.go +++ b/pkg/microservice/aslan/core/workflow/service/webhook/utils.go @@ -18,11 +18,13 @@ package webhook import ( "context" - "encoding/base64" "errors" "fmt" + "os" "path" + "path/filepath" "strings" + "sync" "github.com/google/go-github/v35/github" "github.com/hashicorp/go-multierror" @@ -34,44 +36,53 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/codehub" + environmentservice "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/service/service" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/codehost" - "github.com/koderover/zadig/pkg/shared/poetry" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" e "github.com/koderover/zadig/pkg/tool/errors" githubtool "github.com/koderover/zadig/pkg/tool/git/github" gitlabtool "github.com/koderover/zadig/pkg/tool/git/gitlab" - "github.com/koderover/zadig/pkg/tool/ilyshin" "github.com/koderover/zadig/pkg/tool/kube/serializer" "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types" "github.com/koderover/zadig/pkg/util" ) -func syncContent(args *commonmodels.Service, logger *zap.SugaredLogger) error { +func syncContentFromCodehub(args *commonmodels.Service, logger *zap.SugaredLogger) error { address, _, repo, branch, path, pathType, err := GetOwnerRepoBranchPath(args.SrcPath) if err != nil { logger.Errorf("Failed to parse url %s, err: %s", args.SrcPath, err) return fmt.Errorf("url parse failure, err: %s", err) } + if len(args.LoadPath) > 0 { + path = args.LoadPath + } + if len(args.BranchName) > 0 { + branch = args.BranchName + } + if len(args.RepoName) > 0 { + repo = args.RepoName + } + var yamls []string - switch args.Source { - case setting.SourceFromCodeHub: - client, err := getCodehubClientByAddress(address) - if err != nil { - logger.Errorf("Failed to get codehub client, error: %s", err) - return err - } - repoUUID, err := client.GetRepoUUID(repo) - if err != nil { - logger.Errorf("Failed to get repoUUID, error: %s", err) - return err - } - yamls, err = client.GetYAMLContents(repoUUID, branch, path, pathType == "tree", true) - if err != nil { - logger.Errorf("Failed to get yamls, error: %s", err) - return err - } + client, err := getCodehubClientByAddress(address) + if err != nil { + logger.Errorf("Failed to get codehub client, error: %s", err) + return err + } + repoUUID, err := client.GetRepoUUID(repo) + if err != nil { + logger.Errorf("Failed to get repoUUID, error: %s", err) + return err + } + yamls, err = client.GetYAMLContents(repoUUID, branch, path, pathType == "tree", true) + if err != nil { + logger.Errorf("Failed to get yamls, error: %s", err) + return err } args.KubeYamls = yamls @@ -80,6 +91,24 @@ func syncContent(args *commonmodels.Service, logger *zap.SugaredLogger) error { return nil } +func reloadServiceTmplFromGit(svc *commonmodels.Service, log *zap.SugaredLogger) error { + _, err := service.CreateOrUpdateHelmServiceFromGitRepo(svc.ProductName, &service.HelmServiceCreationArgs{ + HelmLoadSource: service.HelmLoadSource{ + Source: service.LoadFromRepo, + }, + CreatedBy: svc.CreateBy, + CreateFrom: &service.CreateFromRepo{ + CodehostID: svc.CodehostID, + Owner: svc.RepoOwner, + Namespace: svc.GetRepoNamespace(), + Repo: svc.RepoName, + Branch: svc.BranchName, + Paths: []string{svc.LoadPath}, + }, + }, true, log) + return err +} + // fillServiceTmpl 更新服务模板参数 func fillServiceTmpl(userName string, args *commonmodels.Service, log *zap.SugaredLogger) error { if args == nil { @@ -108,18 +137,6 @@ func fillServiceTmpl(userName string, args *commonmodels.Service, log *zap.Sugar log.Errorf("Sync content from gitlab failed, error: %v", err) return err } - } else if args.Source == setting.SourceFromIlyshin { - // Set args.Commit - if err := syncIlyshinLatestCommit(args, log); err != nil { - log.Errorf("Sync change log from ilyshin failed, error: %v", err) - return err - } - // 从 ilyshin 同步 args 指定的 Commit 下，指定目录中对应的 Yaml 文件 - // Set args.Yaml & args.KubeYamls - if err := syncContentFromIlyshin(userName, args); err != nil { - log.Errorf("Sync content from ilyshin failed, error: %v", err) - return err - } } else if args.Source == setting.SourceFromGithub { err := syncContentFromGithub(args, log) if err != nil { @@ -127,7 +144,7 @@ func fillServiceTmpl(userName string, args *commonmodels.Service, log *zap.Sugar return err } } else if args.Source == setting.SourceFromCodeHub { - err := syncContent(args, log) + err := syncContentFromCodehub(args, log) if err != nil { log.Errorf("Sync content from codehub failed, error: %v", err) return err @@ -144,18 +161,40 @@ func fillServiceTmpl(userName string, args *commonmodels.Service, log *zap.Sugar if err := setCurrentContainerImages(args); err != nil { return err } - log.Infof("find %d containers in service %s", len(args.Containers), args.ServiceName) - } - // 设置新的版本号 - serviceTemplate := fmt.Sprintf(setting.ServiceTemplateCounterName, args.ServiceName, args.ProductName) - rev, err := commonrepo.NewCounterColl().GetNextSeq(serviceTemplate) - if err != nil { - return fmt.Errorf("get next service template revision error: %v", err) - } + // generate new revision + serviceTemplate := fmt.Sprintf(setting.ServiceTemplateCounterName, args.ServiceName, args.ProductName) + rev, err := commonrepo.NewCounterColl().GetNextSeq(serviceTemplate) + if err != nil { + return fmt.Errorf("get next service template revision error: %v", err) + } + args.Revision = rev + // update service template + if err := commonrepo.NewServiceColl().Create(args); err != nil { + log.Errorf("Failed to sync service %s from github path %s error: %v", args.ServiceName, args.SrcPath, err) + return e.ErrCreateTemplate.AddDesc(err.Error()) + } + return environmentservice.AutoDeployYamlServiceToEnvs(args.CreateBy, "", args, log) + } else if args.Type == setting.HelmDeployType { + if args.Source == setting.SourceFromGitlab { + // Set args.Commit + if err := syncLatestCommit(args); err != nil { + log.Errorf("Sync change log from gitlab failed, error: %s", err) + return err + } - args.Revision = rev + if err := reloadServiceTmplFromGit(args, log); err != nil { + log.Errorf("Sync content from gitlab failed, error: %s", err) + return err + } + } else if args.Source == setting.SourceFromGithub { + if err := reloadServiceTmplFromGit(args, log); err != nil { + log.Errorf("Sync content from github failed, error: %s", err) + return err + } + } + } return nil } @@ -165,50 +204,41 @@ func syncLatestCommit(service *commonmodels.Service) error { return fmt.Errorf("url不能是空的") } - address, owner, repo, branch, path, _, err := GetOwnerRepoBranchPath(service.SrcPath) + _, owner, repo, branch, path, _, err := GetOwnerRepoBranchPath(service.SrcPath) if err != nil { return fmt.Errorf("url 必须包含 owner/repo/tree/branch/path，具体请参考 Placeholder 提示") } - client, err := getGitlabClientByAddress(address) + client, err := getGitlabClientByCodehostId(service.CodehostID) if err != nil { return err } - commit, err := GitlabGetLatestCommit(client, owner, repo, branch, path) - if err != nil { - return err + if len(service.BranchName) > 0 { + branch = service.BranchName } - service.Commit = &commonmodels.Commit{ - SHA: commit.ID, - Message: commit.Message, + if len(service.LoadPath) > 0 { + path = service.LoadPath } - return nil -} - -func syncIlyshinLatestCommit(service *commonmodels.Service, log *zap.SugaredLogger) error { - if service.SrcPath == "" { - return fmt.Errorf("url不能是空的") + if len(service.GetRepoNamespace()) > 0 { + owner = service.GetRepoNamespace() } - - address, owner, repo, branch, path, _, err := GetOwnerRepoBranchPath(service.SrcPath) - if err != nil { - return fmt.Errorf("url 必须包含 owner/repo/tree/branch/path，具体请参考 Placeholder 提示") + if len(service.RepoName) > 0 { + repo = service.RepoName } - client, err := getIlyshinClientByAddress(address) + commit, err := GitlabGetLatestCommit(client, owner, repo, branch, path) if err != nil { return err } - commit, err := client.GetLatestCommit(owner, repo, branch, path, log) - if err != nil { - return err - } - service.Commit = &commonmodels.Commit{ - SHA: commit.ID, - Message: commit.Message, + if commit != nil { + service.Commit = &commonmodels.Commit{ + SHA: commit.ID, + Message: commit.Message, + } } + return nil } @@ -239,46 +269,46 @@ func syncCodehubLatestCommit(service *commonmodels.Service) error { } func getCodehubClientByAddress(address string) (*codehub.Client, error) { - opt := &codehost.Option{ + opt := &systemconfig.Option{ Address: address, - CodeHostType: codehost.CodeHubProvider, + CodeHostType: systemconfig.CodeHubProvider, } - codehost, err := codehost.GetCodeHostInfo(opt) + codehost, err := systemconfig.GetCodeHostInfo(opt) if err != nil { log.Error(err) return nil, e.ErrCodehostListProjects.AddDesc("git client is nil") } - client := codehub.NewClient(codehost.AccessKey, codehost.SecretKey, codehost.Region) + client := codehub.NewClient(codehost.AccessKey, codehost.SecretKey, codehost.Region, config.ProxyHTTPSAddr(), codehost.EnableProxy) return client, nil } -func getIlyshinClientByAddress(address string) (*ilyshin.Client, error) { - opt := &codehost.Option{ - Address: address, - CodeHostType: codehost.IlyshinProvider, +func getGitlabClientByCodehostId(codehostId int) (*gitlabtool.Client, error) { + codehost, err := systemconfig.New().GetCodeHost(codehostId) + if err != nil { + log.Error(err) + return nil, e.ErrCodehostListProjects.AddDesc(fmt.Sprintf("failed to get codehost:%d, err: %s", codehost, err)) } - codehost, err := codehost.GetCodeHostInfo(opt) + client, err := gitlabtool.NewClient(codehost.ID, codehost.Address, codehost.AccessToken, config.ProxyHTTPSAddr(), codehost.EnableProxy) if err != nil { log.Error(err) - return nil, e.ErrCodehostListProjects.AddDesc("git client is nil") + return nil, e.ErrCodehostListProjects.AddDesc(err.Error()) } - client := ilyshin.NewClient(codehost.Address, codehost.AccessToken) return client, nil } func getGitlabClientByAddress(address string) (*gitlabtool.Client, error) { - opt := &codehost.Option{ + opt := &systemconfig.Option{ Address: address, - CodeHostType: codehost.GitLabProvider, + CodeHostType: systemconfig.GitLabProvider, } - codehost, err := codehost.GetCodeHostInfo(opt) + codehost, err := systemconfig.GetCodeHostInfo(opt) if err != nil { log.Error(err) return nil, e.ErrCodehostListProjects.AddDesc("git client is nil") } - client, err := gitlabtool.NewClient(codehost.Address, codehost.AccessToken) + client, err := gitlabtool.NewClient(codehost.ID, codehost.Address, codehost.AccessToken, config.ProxyHTTPSAddr(), codehost.EnableProxy) if err != nil { log.Error(err) return nil, e.ErrCodehostListProjects.AddDesc(err.Error()) @@ -353,12 +383,13 @@ func syncContentFromGitlab(userName string, args *commonmodels.Service) error { return nil } - address, owner, repo, branch, path, pathType, err := GetOwnerRepoBranchPath(args.SrcPath) - if err != nil { - return fmt.Errorf("url format failed") + var owner, repo, branch, path string = args.GetRepoNamespace(), args.RepoName, args.BranchName, args.LoadPath + var pathType = "tree" + if strings.Contains(args.SrcPath, "blob") { + pathType = "blob" } - client, err := getGitlabClientByAddress(address) + client, err := getGitlabClientByCodehostId(args.CodehostID) if err != nil { return err } @@ -384,99 +415,21 @@ func joinYamls(files []string) string { return strings.Join(files, setting.YamlFileSeperator) } -// IlyshinGetRawFiles ... -func IlyshinGetRawFiles(client *ilyshin.Client, owner, repo, ref, path, pathType string) (files []string, err error) { - files = make([]string, 0) - var errs *multierror.Error - if pathType == "tree" { - nodes, err := client.ListTree(owner, repo, ref, path) - if err != nil { - return files, err - } - for _, node := range nodes { - // if node type is "tree", it is a directory, skip it for now - if node.Type == "tree" { - continue - } - fileName := strings.ToLower(node.Name) - if !strings.HasSuffix(fileName, ".yaml") && !strings.HasSuffix(fileName, ".yml") { - continue - } - // if node type is "blob", it is a file - // Path is filepath of a node - content, err := client.GetRawFile(owner, repo, ref, node.Path) - if err != nil { - errs = multierror.Append(errs, err) - } - contentStr := string(content) - contentStr = util.ReplaceWrapLine(contentStr) - files = append(files, contentStr) - } - return files, errs.ErrorOrNil() - } - fileInfo, err := client.GetFile(owner, repo, ref, path) - if err != nil { - return files, err - } - decodedFile, err := base64.StdEncoding.DecodeString(fileInfo.Content) - if err != nil { - return files, err - } - files = append(files, string(decodedFile)) - return files, errs.ErrorOrNil() -} - -// syncContentFromIlyshin ... -// sync content with commit, args.Commit should not be nil -func syncContentFromIlyshin(userName string, args *commonmodels.Service) error { - if args.Commit == nil { - return nil - } - - address, owner, repo, branch, path, pathType, err := GetOwnerRepoBranchPath(args.SrcPath) - if err != nil { - return fmt.Errorf("url format failed") - } - - client, err := getIlyshinClientByAddress(address) - if err != nil { - return err - } - - files, err := IlyshinGetRawFiles(client, owner, repo, branch, path, pathType) - if err != nil { - return err - } - if userName != setting.WebhookTaskCreator { - if len(files) == 0 { - return fmt.Errorf("没有检索到yml,yaml类型文件，请检查目录是否正确") - } - } - // KubeYamls field is dynamicly synced. - // 根据gitlab sync的内容来设置args.KubeYamls - args.KubeYamls = files - // 拼装并设置args.Yaml - args.Yaml = joinYamls(files) - return nil -} - func syncContentFromGithub(args *commonmodels.Service, log *zap.SugaredLogger) error { // 根据pipeline中的filepath获取文件内容 - address, owner, repo, branch, path, _, err := GetOwnerRepoBranchPath(args.SrcPath) - if err != nil { - log.Errorf("GetOwnerRepoBranchPath failed, srcPath:%s, err:%v", args.SrcPath, err) - return errors.New("invalid url " + args.SrcPath) - } + var owner, repo, branch, path = args.GetRepoNamespace(), args.RepoName, args.BranchName, args.LoadPath - ch, err := codehost.GetCodeHostInfo( - &codehost.Option{CodeHostType: poetry.GitHubProvider, Address: address, Namespace: owner}) + ch, err := systemconfig.New().GetCodeHost(args.CodehostID) if err != nil { - log.Errorf("GetCodeHostInfo failed, srcPath:%s, err:%v", args.SrcPath, err) + log.Errorf("failed to getCodeHostInfo, srcPath:%s, err:%s", args.SrcPath, err) return err } gc := githubtool.NewClient(&githubtool.Config{AccessToken: ch.AccessToken, Proxy: config.ProxyHTTPSAddr()}) fileContent, directoryContent, err := gc.GetContents(context.TODO(), owner, repo, path, &github.RepositoryContentGetOptions{Ref: branch}) + if err != nil { + return err + } if fileContent != nil { svcContent, _ := fileContent.GetContent() splitYaml := SplitYaml(svcContent) @@ -623,7 +576,7 @@ func (m MatchFolders) ContainsFile(file string) bool { return false } -func MatchChanges(m commonmodels.MainHookRepo, files []string) bool { +func MatchChanges(m *commonmodels.MainHookRepo, files []string) bool { mf := MatchFolders(m.MatchFolders) for _, file := range files { if matches := mf.ContainsFile(file); matches { @@ -633,7 +586,19 @@ func MatchChanges(m commonmodels.MainHookRepo, files []string) bool { return false } -func EventConfigured(m commonmodels.MainHookRepo, event config.HookEventType) bool { +func ConvertScanningHookToMainHookRepo(hook *types.ScanningHook) *commonmodels.MainHookRepo { + return &commonmodels.MainHookRepo{ + Source: hook.Source, + RepoOwner: hook.RepoOwner, + RepoName: hook.RepoName, + Branch: hook.Branch, + MatchFolders: hook.MatchFolders, + CodehostID: hook.CodehostID, + Events: hook.Events, + } +} + +func EventConfigured(m *commonmodels.MainHookRepo, event config.HookEventType) bool { for _, ev := range m.Events { if ev == event { return true @@ -642,3 +607,165 @@ func EventConfigured(m commonmodels.MainHookRepo, event config.HookEventType) bo return false } + +func ServicesMatchChangesFiles(mf *MatchFoldersElem, files []string) []BuildServices { + resMactchSvr := []BuildServices{} + var wg sync.WaitGroup + var mutex sync.Mutex + for _, mftreeElem := range mf.MatchFoldersTree { + wg.Add(1) + go func(mftree *MatchFoldersTree) { + defer wg.Done() + mf := MatchFolders(mftree.FileTree) + for _, file := range files { + if matches := mf.ContainsFile(file); matches { + mutex.Lock() + resMactchSvr = append(resMactchSvr, BuildServices{Name: mftree.Name, ServiceModule: mftree.ServiceModule}) + mutex.Unlock() + break + } + } + }(mftreeElem) + } + wg.Wait() + return resMactchSvr +} + +func getServiceTypeByProject(productName string) (string, error) { + projectInfo, err := templaterepo.NewProductColl().Find(productName) + if err != nil { + return "", err + } + projectType := setting.K8SDeployType + if projectInfo == nil || projectInfo.ProductFeature == nil { + return projectType, nil + } else if projectInfo.ProductFeature.BasicFacility == setting.BasicFacilityK8S { + if projectInfo.ProductFeature.DeployType == setting.HelmDeployType { + return setting.HelmDeployType, nil + } + return projectType, nil + } else if projectInfo.ProductFeature.BasicFacility == setting.BasicFacilityCVM { + return setting.PMDeployType, nil + } + return projectType, nil +} + +func existStage(expectStage Stage, triggerYaml *TriggerYaml) bool { + for _, stage := range triggerYaml.Stages { + if stage == expectStage { + return true + } + } + return false +} + +func checkTriggerYamlParams(triggerYaml *TriggerYaml) error { + //check stages + for _, stage := range triggerYaml.Stages { + if stage != StageBuild && stage != StageDeploy && stage != StageTest { + return fmt.Errorf("stages must %s or %s or %s", StageBuild, StageDeploy, StageTest) + } + } + + //check build + if len(triggerYaml.Build) == 0 { + return errors.New("build is empty") + } + for _, bd := range triggerYaml.Build { + if bd.Name == "" || bd.ServiceModule == "" { + return errors.New("build.name or build.service_module is empty") + } + } + + //check deploy + if triggerYaml.Deploy == nil { + return errors.New("deploy is empty") + } + if len(triggerYaml.Deploy.Envsname) == 0 { + return errors.New("deploy.envs_name is empty") + } + if triggerYaml.Deploy.Strategy != DeployStrategySingle && triggerYaml.Deploy.Strategy != DeployStrategyBase && triggerYaml.Deploy.Strategy != DeployStrategyDynamic { + return fmt.Errorf("deploy.strategy must %s or %s or %s", DeployStrategySingle, DeployStrategyDynamic, DeployStrategyBase) + } + if triggerYaml.Deploy.Strategy == DeployStrategyBase { + if triggerYaml.Deploy.BaseNamespace == "" { + return errors.New("deploy.base_env is empty") + } + if triggerYaml.Deploy.EnvRecyclePolicy != EnvRecyclePolicySuccess && triggerYaml.Deploy.EnvRecyclePolicy != EnvRecyclePolicyAlways && triggerYaml.Deploy.EnvRecyclePolicy != EnvRecyclePolicyNever { + return errors.New("deploy.env_recycle_policy must success/always/never") + } + } else { + if triggerYaml.Deploy.BaseNamespace != "" { + return errors.New("deploy.base_env must empty") + } + } + + //check test + if existStage(StageTest, triggerYaml) { + if len(triggerYaml.Test) == 0 { + return errors.New("test is empty") + } + for _, tt := range triggerYaml.Test { + if tt.Repo == nil { + return errors.New("test.repo.strategy must default/currentRepo") + } + if tt.Repo.Strategy != TestRepoStrategyDefault && tt.Repo.Strategy != TestRepoStrategyCurrentRepo { + return errors.New("test.repo.strategy must default/currentRepo") + } + } + } + + //check rule + if triggerYaml.Rules == nil { + return errors.New("rules must exist") + } + if len(triggerYaml.Rules.Branchs) == 0 { + return errors.New("rules.branchs must exist") + } + for _, ev := range triggerYaml.Rules.Events { + if ev != "pull_request" && ev != "push" && ev != "tag" { + return errors.New("rules.event must be pull_request or push or tag") + } + } + if triggerYaml.Rules.MatchFolders == nil { + return errors.New("rules.match_folders must exist") + } + for _, mf := range triggerYaml.Rules.MatchFolders.MatchFoldersTree { + if mf.Name == "" || mf.ServiceModule == "" { + return errors.New("match_folders.match_folders_tree.name or match_folders.match_folders_tree.service_module is empty") + } + if len(mf.FileTree) == 0 { + return errors.New("match_folders.match_folders_tree.file_tree is empty") + } + } + + return nil +} + +func getServiceSrcPath(service *commonmodels.Service) (string, error) { + if service.LoadPath != "" { + return service.LoadPath, nil + } + _, _, _, _, p, _, err := GetOwnerRepoBranchPath(service.SrcPath) + return p, err +} + +func checkRepoNamespaceMatch(hookRepo *commonmodels.MainHookRepo, pathWithNamespace string) bool { + return (hookRepo.GetRepoNamespace() + "/" + hookRepo.RepoName) == pathWithNamespace +} + +// check if sub path is a part of parent path +// eg: parent: k1/k2 sub: k1/k2/k3 return true +// parent k1/k2-2 sub: k1/k2/k3 return false +func subElem(parent, sub string) bool { + up := ".." + string(os.PathSeparator) + rel, err := filepath.Rel(parent, sub) + if err != nil { + log.Errorf("failed to check path is relative, parent: %s, sub: %s", parent, sub) + return false + } + if !strings.HasPrefix(rel, up) && rel != ".." { + return true + } + return false +} diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/artifact_task.go b/pkg/microservice/aslan/core/workflow/service/workflow/artifact_task.go new file mode 100644 index 0000000000000000000000000000000000000000..2be1fb3843e795f8214efa0b456812be955e5b40 --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/service/workflow/artifact_task.go @@ -0,0 +1,133 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package workflow + +import ( + "fmt" + "sort" + + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + taskmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3" + "github.com/koderover/zadig/pkg/setting" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +// get global config payload +func CreateArtifactPackageTask(args *commonmodels.ArtifactPackageTaskArgs, taskCreator string, log *zap.SugaredLogger) (int64, error) { + configPayload := commonservice.GetConfigPayload(0) + repos, err := commonservice.ListRegistryNamespaces("", true, log) + + if err != nil { + log.Errorf("CreateArtifactPackageTask query registries failed, err: %s", err) + return 0, fmt.Errorf("failed to query registries") + } + + registriesInvolved := sets.NewString() + registriesInvolved.Insert(args.SourceRegistries...) + registriesInvolved.Insert(args.TargetRegistries...) + + configPayload.RepoConfigs = make(map[string]*commonmodels.RegistryNamespace) + for _, repo := range repos { + if !registriesInvolved.Has(repo.ID.Hex()) { + continue + } + configPayload.RepoConfigs[repo.ID.Hex()] = repo + } + + defaultS3, err := s3.FindDefaultS3() + if err != nil { + err = e.ErrFindDefaultS3Storage.AddDesc("default storage is required by distribute task") + return 0, err + } + + defaultURL, err := defaultS3.GetEncryptedURL() + if err != nil { + err = e.ErrS3Storage.AddErr(err) + return 0, err + } + + task := &taskmodels.Task{ + Type: config.ArtifactType, + ProductName: args.ProjectName, + Status: config.StatusCreated, + ArtifactPackageTaskArgs: args, + TaskCreator: taskCreator, + ConfigPayload: configPayload, + StorageURI: defaultURL, + } + + subTask, err := (&taskmodels.ArtifactPackage{ + TaskType: config.TaskArtifactPackage, + Enabled: true, + TaskStatus: "", + Timeout: 0, + StartTime: 0, + EndTime: 0, + LogFile: "", + Images: args.Images, + SourceRegistries: args.SourceRegistries, + TargetRegistries: args.TargetRegistries, + }).ToSubTask() + + if err != nil { + return 0, err + } + + task.SubTasks = []map[string]interface{}{subTask} + + if err := ensurePipelineTask(&taskmodels.TaskOpt{ + Task: task, + }, log); err != nil { + log.Errorf("CreateServiceTask ensurePipelineTask err : %v", err) + return 0, err + } + + stages := make([]*commonmodels.Stage, 0) + for _, subTask := range task.SubTasks { + AddSubtaskToStage(&stages, subTask, args.EnvName) + } + sort.Sort(ByStageKind(stages)) + task.Stages = stages + if len(task.Stages) == 0 { + return 0, e.ErrCreateTask.AddDesc(e.PipelineSubTaskNotFoundErrMsg) + } + + pipelineName := fmt.Sprintf("%s-%s-%s", args.ProjectName, args.EnvName, "artifact") + nextTaskID, err := commonrepo.NewCounterColl().GetNextSeq(fmt.Sprintf(setting.ServiceTaskFmt, pipelineName)) + if err != nil { + log.Errorf("CreateServiceTask Counter.GetNextSeq error: %v", err) + return 0, e.ErrGetCounter.AddDesc(err.Error()) + } + + task.SubTasks = []map[string]interface{}{} + task.TaskID = nextTaskID + task.PipelineName = pipelineName + + if err := CreateTask(task); err != nil { + log.Error(err) + return 0, e.ErrCreateTask + } + + return nextTaskID, nil +} diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/bundle.go b/pkg/microservice/aslan/core/workflow/service/workflow/bundle.go new file mode 100644 index 0000000000000000000000000000000000000000..c85e45cd9dd0e570a0d3d9275f03bc6a8d72fd3f --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/service/workflow/bundle.go @@ -0,0 +1,73 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package workflow + +import ( + "go.uber.org/zap" + + commonConfig "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/label/config" + labeldb "github.com/koderover/zadig/pkg/microservice/aslan/core/label/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/label/service" + "github.com/koderover/zadig/pkg/tool/log" +) + +type resourceSpec struct { + ResourceID string `json:"resourceID"` + ProjectName string `json:"projectName"` + Spec []string `json:"spec"` +} + +func GetBundleResources(logger *zap.SugaredLogger) ([]*resourceSpec, error) { + var res []*resourceSpec + workflows, err := mongodb.NewWorkflowColl().List(&mongodb.ListWorkflowOption{}) + if err != nil { + log.Error("Failed to list workflows , err:%s", err) + return nil, err + } + + // get labels by workflow resources ids + var resources []labeldb.Resource + for _, workflow := range workflows { + resource := labeldb.Resource{ + Name: workflow.Name, + ProjectName: workflow.ProductTmplName, + Type: string(config.ResourceTypeWorkflow), + } + resources = append(resources, resource) + } + labelsResp, err := service.ListLabelsByResources(resources, logger) + if err != nil { + return nil, err + } + + for _, workflow := range workflows { + resourceKey := commonConfig.BuildResourceKey(string(config.ResourceTypeWorkflow), workflow.ProductTmplName, workflow.Name) + resourceSpec := &resourceSpec{ + ResourceID: workflow.Name, + ProjectName: workflow.ProductTmplName, + } + if labels, ok := labelsResp.Labels[resourceKey]; ok { + for _, v := range labels { + resourceSpec.Spec = append(resourceSpec.Spec, v.Key+":"+v.Value) + } + } + res = append(res, resourceSpec) + } + return res, nil +} diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/convert.go b/pkg/microservice/aslan/core/workflow/service/workflow/convert.go index 8116274260d8665e29a909c85358ab53a6a029da..457e5d513325bc7082ec07e9103d55170b69d9b1 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/convert.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/convert.go @@ -25,103 +25,108 @@ import ( commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/types" ) func ConvertQueueToTask(queueTask *commonmodels.Queue) *task.Task { return &task.Task{ - TaskID: queueTask.TaskID, - ProductName: queueTask.ProductName, - PipelineName: queueTask.PipelineName, - Type: queueTask.Type, - Status: queueTask.Status, - Description: queueTask.Description, - TaskCreator: queueTask.TaskCreator, - TaskRevoker: queueTask.TaskRevoker, - CreateTime: queueTask.CreateTime, - StartTime: queueTask.StartTime, - EndTime: queueTask.EndTime, - SubTasks: queueTask.SubTasks, - Stages: queueTask.Stages, - ReqID: queueTask.ReqID, - AgentHost: queueTask.AgentHost, - DockerHost: queueTask.DockerHost, - TeamName: queueTask.TeamName, - IsDeleted: queueTask.IsDeleted, - IsArchived: queueTask.IsArchived, - AgentID: queueTask.AgentID, - MultiRun: queueTask.MultiRun, - Target: queueTask.Target, - BuildModuleVer: queueTask.BuildModuleVer, - ServiceName: queueTask.ServiceName, - TaskArgs: queueTask.TaskArgs, - WorkflowArgs: queueTask.WorkflowArgs, - TestArgs: queueTask.TestArgs, - ServiceTaskArgs: queueTask.ServiceTaskArgs, - ConfigPayload: queueTask.ConfigPayload, - Error: queueTask.Error, - OrgID: queueTask.OrgID, - Services: queueTask.Services, - Render: queueTask.Render, - StorageURI: queueTask.StorageURI, - TestReports: queueTask.TestReports, - RwLock: queueTask.RwLock, - ResetImage: queueTask.ResetImage, - TriggerBy: queueTask.TriggerBy, - Features: queueTask.Features, - IsRestart: queueTask.IsRestart, - StorageEndpoint: queueTask.StorageEndpoint, + TaskID: queueTask.TaskID, + ProductName: queueTask.ProductName, + PipelineName: queueTask.PipelineName, + Type: queueTask.Type, + Status: queueTask.Status, + Description: queueTask.Description, + TaskCreator: queueTask.TaskCreator, + TaskRevoker: queueTask.TaskRevoker, + CreateTime: queueTask.CreateTime, + StartTime: queueTask.StartTime, + EndTime: queueTask.EndTime, + SubTasks: queueTask.SubTasks, + Stages: queueTask.Stages, + ReqID: queueTask.ReqID, + AgentHost: queueTask.AgentHost, + DockerHost: queueTask.DockerHost, + TeamName: queueTask.TeamName, + IsDeleted: queueTask.IsDeleted, + IsArchived: queueTask.IsArchived, + AgentID: queueTask.AgentID, + MultiRun: queueTask.MultiRun, + Target: queueTask.Target, + BuildModuleVer: queueTask.BuildModuleVer, + ServiceName: queueTask.ServiceName, + TaskArgs: queueTask.TaskArgs, + WorkflowArgs: queueTask.WorkflowArgs, + TestArgs: queueTask.TestArgs, + ServiceTaskArgs: queueTask.ServiceTaskArgs, + ArtifactPackageTaskArgs: queueTask.ArtifactPackageTaskArgs, + ConfigPayload: queueTask.ConfigPayload, + Error: queueTask.Error, + Services: queueTask.Services, + Render: queueTask.Render, + StorageURI: queueTask.StorageURI, + TestReports: queueTask.TestReports, + RwLock: queueTask.RwLock, + ResetImage: queueTask.ResetImage, + ResetImagePolicy: queueTask.ResetImagePolicy, + TriggerBy: queueTask.TriggerBy, + Features: queueTask.Features, + IsRestart: queueTask.IsRestart, + StorageEndpoint: queueTask.StorageEndpoint, } } func ConvertTaskToQueue(task *task.Task) *commonmodels.Queue { return &commonmodels.Queue{ - TaskID: task.TaskID, - ProductName: task.ProductName, - PipelineName: task.PipelineName, - Type: task.Type, - Status: task.Status, - Description: task.Description, - TaskCreator: task.TaskCreator, - TaskRevoker: task.TaskRevoker, - CreateTime: task.CreateTime, - StartTime: task.StartTime, - EndTime: task.EndTime, - SubTasks: task.SubTasks, - Stages: task.Stages, - ReqID: task.ReqID, - AgentHost: task.AgentHost, - DockerHost: task.DockerHost, - TeamName: task.TeamName, - IsDeleted: task.IsDeleted, - IsArchived: task.IsArchived, - AgentID: task.AgentID, - MultiRun: task.MultiRun, - Target: task.Target, - BuildModuleVer: task.BuildModuleVer, - ServiceName: task.ServiceName, - TaskArgs: task.TaskArgs, - WorkflowArgs: task.WorkflowArgs, - TestArgs: task.TestArgs, - ServiceTaskArgs: task.ServiceTaskArgs, - ConfigPayload: task.ConfigPayload, - Error: task.Error, - OrgID: task.OrgID, - Services: task.Services, - Render: task.Render, - StorageURI: task.StorageURI, - TestReports: task.TestReports, - RwLock: task.RwLock, - ResetImage: task.ResetImage, - TriggerBy: task.TriggerBy, - Features: task.Features, - IsRestart: task.IsRestart, - StorageEndpoint: task.StorageEndpoint, + TaskID: task.TaskID, + ProductName: task.ProductName, + PipelineName: task.PipelineName, + Type: task.Type, + Status: task.Status, + Description: task.Description, + TaskCreator: task.TaskCreator, + TaskRevoker: task.TaskRevoker, + CreateTime: task.CreateTime, + StartTime: task.StartTime, + EndTime: task.EndTime, + SubTasks: task.SubTasks, + Stages: task.Stages, + ReqID: task.ReqID, + AgentHost: task.AgentHost, + DockerHost: task.DockerHost, + TeamName: task.TeamName, + IsDeleted: task.IsDeleted, + IsArchived: task.IsArchived, + AgentID: task.AgentID, + MultiRun: task.MultiRun, + Target: task.Target, + BuildModuleVer: task.BuildModuleVer, + ServiceName: task.ServiceName, + TaskArgs: task.TaskArgs, + WorkflowArgs: task.WorkflowArgs, + TestArgs: task.TestArgs, + ServiceTaskArgs: task.ServiceTaskArgs, + ArtifactPackageTaskArgs: task.ArtifactPackageTaskArgs, + ConfigPayload: task.ConfigPayload, + Error: task.Error, + Services: task.Services, + Render: task.Render, + StorageURI: task.StorageURI, + TestReports: task.TestReports, + RwLock: task.RwLock, + ResetImage: task.ResetImage, + ResetImagePolicy: task.ResetImagePolicy, + TriggerBy: task.TriggerBy, + Features: task.Features, + IsRestart: task.IsRestart, + StorageEndpoint: task.StorageEndpoint, } } type JenkinsBuildOption struct { + BuildName string Version string Target string ServiceName string @@ -135,6 +140,7 @@ func JenkinsBuildModuleToSubTasks(jenkinsBuildOption *JenkinsBuildOption, log *z ) opt := &commonrepo.BuildListOption{ + Name: jenkinsBuildOption.BuildName, ServiceName: jenkinsBuildOption.ServiceName, ProductName: jenkinsBuildOption.ProductName, } @@ -148,36 +154,43 @@ func JenkinsBuildModuleToSubTasks(jenkinsBuildOption *JenkinsBuildOption, log *z return nil, e.ErrConvertSubTasks.AddErr(err) } - jenkinsIntegrations, err := commonrepo.NewJenkinsIntegrationColl().List() + registries, err := commonservice.ListRegistryNamespaces("", true, log) if err != nil { return nil, e.ErrConvertSubTasks.AddErr(err) } - jenkinsBuildParams := make([]*task.JenkinsBuildParam, 0) + jenkinsBuildParams := make([]*types.JenkinsBuildParam, 0) for _, jenkinsBuildParam := range jenkinsBuildOption.JenkinsBuildArgs.JenkinsBuildParams { - jenkinsBuildParams = append(jenkinsBuildParams, &task.JenkinsBuildParam{ - Name: jenkinsBuildParam.Name, - Value: jenkinsBuildParam.Value, + jenkinsBuildParams = append(jenkinsBuildParams, &types.JenkinsBuildParam{ + Name: jenkinsBuildParam.Name, + Value: jenkinsBuildParam.Value, + AutoGenerate: jenkinsBuildParam.AutoGenerate, }) } for _, module := range modules { + jenkinsIntegration := getJenkinsIntegration(module.JenkinsBuild) + if jenkinsIntegration == nil { + return nil, e.ErrConvertSubTasks.AddDesc("not found jenkins client") + } build := &task.JenkinsBuild{ TaskType: config.TaskJenkinsBuild, Enabled: true, ServiceName: jenkinsBuildOption.Target, Service: jenkinsBuildOption.ServiceName, ResReq: module.PreBuild.ResReq, + ResReqSpec: module.PreBuild.ResReqSpec, Timeout: module.Timeout, JenkinsBuildArgs: &task.JenkinsBuildArgs{ JobName: jenkinsBuildOption.JenkinsBuildArgs.JobName, JenkinsBuildParams: jenkinsBuildParams, }, JenkinsIntegration: &task.JenkinsIntegration{ - URL: jenkinsIntegrations[0].URL, - Username: jenkinsIntegrations[0].Username, - Password: jenkinsIntegrations[0].Password, + URL: jenkinsIntegration.URL, + Username: jenkinsIntegration.Username, + Password: jenkinsIntegration.Password, }, + Registries: registries, } bst, err := build.ToSubTask() @@ -190,6 +203,17 @@ func JenkinsBuildModuleToSubTasks(jenkinsBuildOption *JenkinsBuildOption, log *z return subTasks, nil } +func getJenkinsIntegration(jenkinsBuild *commonmodels.JenkinsBuild) *commonmodels.JenkinsIntegration { + if jenkinsBuild == nil { + return nil + } + jenkinsIntegration, err := commonrepo.NewJenkinsIntegrationColl().Get(jenkinsBuild.JenkinsID) + if err != nil { + return nil + } + return jenkinsIntegration +} + func AddPipelineJiraSubTask(pipeline *commonmodels.Pipeline, log *zap.SugaredLogger) (map[string]interface{}, error) { jira := &task.Jira{ TaskType: config.TaskJira, diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/gerrit.go b/pkg/microservice/aslan/core/workflow/service/workflow/gerrit.go index b5be0c5edd1078286fe01d64f4d5553dff5e97a8..dae225e4c46e8138ee856a0217f5d75a74246156 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/gerrit.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/gerrit.go @@ -24,7 +24,7 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/codehost" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" "github.com/koderover/zadig/pkg/tool/gerrit" "github.com/koderover/zadig/pkg/tool/httpclient" ) @@ -36,10 +36,7 @@ func CreateGerritWebhook(workflow *commonmodels.Workflow, log *zap.SugaredLogger continue } - opt := &codehost.Option{ - CodeHostID: workflowWebhook.MainRepo.CodehostID, - } - detail, err := codehost.GetCodeHostInfo(opt) + detail, err := systemconfig.New().GetCodeHost(workflowWebhook.MainRepo.CodehostID) if err != nil { return err } @@ -88,10 +85,7 @@ func UpdateGerritWebhook(currentWorkflow *commonmodels.Workflow, log *zap.Sugare continue } - opt := &codehost.Option{ - CodeHostID: oldWorkflowWebhook.MainRepo.CodehostID, - } - detail, err := codehost.GetCodeHostInfo(opt) + detail, err := systemconfig.New().GetCodeHost(oldWorkflowWebhook.MainRepo.CodehostID) if err != nil { return err } @@ -113,10 +107,7 @@ func UpdateGerritWebhook(currentWorkflow *commonmodels.Workflow, log *zap.Sugare continue } - opt := &codehost.Option{ - CodeHostID: workflowWebhook.MainRepo.CodehostID, - } - detail, err := codehost.GetCodeHostInfo(opt) + detail, err := systemconfig.New().GetCodeHost(workflowWebhook.MainRepo.CodehostID) if err != nil { return err } @@ -155,10 +146,7 @@ func DeleteGerritWebhook(workflow *commonmodels.Workflow, log *zap.SugaredLogger continue } - opt := &codehost.Option{ - CodeHostID: workflowWebhook.MainRepo.CodehostID, - } - detail, err := codehost.GetCodeHostInfo(opt) + detail, err := systemconfig.New().GetCodeHost(workflowWebhook.MainRepo.CodehostID) if err != nil { return err } diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/github.go b/pkg/microservice/aslan/core/workflow/service/workflow/github.go index 4a93821185fac319d3a2c4d2702bb2d46a5c44cc..4b7ca68b6e6137fcbd74b6ee8b10ea2d94b75943 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/github.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/github.go @@ -27,7 +27,7 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/github" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/codehost" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" e "github.com/koderover/zadig/pkg/tool/errors" ) @@ -85,12 +85,12 @@ func createGitCheck(pt *task.Task, log *zap.SugaredLogger) error { } log.Infof("Init GitHub status") - ch, err := codehost.GetCodeHostInfoByID(pt.TriggerBy.CodehostID) + ch, err := systemconfig.New().GetCodeHost(pt.TriggerBy.CodehostID) if err != nil { - log.Errorf("GetCodeHostInfoByID failed, err:%v", err) + log.Errorf("Failed to get codeHost, err:%v", err) return e.ErrGithubUpdateStatus.AddErr(err) } - gc := github.NewClient(ch.AccessToken, config.ProxyHTTPSAddr()) + gc := github.NewClient(ch.AccessToken, config.ProxyHTTPSAddr(), ch.EnableProxy) return gc.UpdateCheckStatus(&github.StatusOptions{ Owner: hook.Owner, diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/gitlab.go b/pkg/microservice/aslan/core/workflow/service/workflow/gitlab.go index 58fc471b69e5231f663e9853a95a8d0190ed6e88..401aaf61f3aa89d8e619dd86197474a0fa16d119 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/gitlab.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/gitlab.go @@ -19,16 +19,18 @@ package workflow import ( "errors" "fmt" + "net/http" + "net/url" "sort" "time" + "github.com/koderover/zadig/pkg/microservice/aslan/config" "github.com/xanzy/go-gitlab" "go.uber.org/zap" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/codehost" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" "github.com/koderover/zadig/pkg/tool/gerrit" - "github.com/koderover/zadig/pkg/tool/ilyshin" ) // RepoCommit : Repository commit struct @@ -41,18 +43,25 @@ type RepoCommit struct { } func QueryByBranch(id int, owner, name, branch string, log *zap.SugaredLogger) (*RepoCommit, error) { - opt := &codehost.Option{ - CodeHostID: id, - } - ch, err := codehost.GetCodeHostInfo(opt) + ch, err := systemconfig.New().GetCodeHost(id) if err != nil { return nil, err } if ch.Type == setting.SourceFromGitlab { token, address := ch.AccessToken, ch.Address - - cli, err := gitlab.NewOAuthClient(token, gitlab.WithBaseURL(address)) + var client *http.Client + if ch.EnableProxy { + proxyURL, err := url.Parse(config.ProxyHTTPSAddr()) + if err != nil { + return nil, err + } + transport := &http.Transport{Proxy: http.ProxyURL(proxyURL)} + client = &http.Client{Transport: transport} + } else { + client = http.DefaultClient + } + cli, err := gitlab.NewOAuthClient(token, gitlab.WithBaseURL(address), gitlab.WithHTTPClient(client)) if err != nil { return nil, fmt.Errorf("set base url failed, err:%v", err) } @@ -70,7 +79,7 @@ func QueryByBranch(id int, owner, name, branch string, log *zap.SugaredLogger) ( Message: br.Commit.Message, }, nil } else if ch.Type == setting.SourceFromGerrit { - cli := gerrit.NewClient(ch.Address, ch.AccessToken) + cli := gerrit.NewClient(ch.Address, ch.AccessToken, config.ProxyHTTPSAddr(), ch.EnableProxy) commit, err := cli.GetCommitByBranch(name, branch) if err != nil { return nil, err @@ -85,33 +94,13 @@ func QueryByBranch(id int, owner, name, branch string, log *zap.SugaredLogger) ( CreatedAt: &commitDate, Message: commit.Message, }, nil - } else if ch.Type == setting.SourceFromIlyshin { - client := ilyshin.NewClient(ch.Address, ch.AccessToken) - branches, err := client.ListBranches(owner, name, log) - if err != nil { - return nil, err - } - for _, br := range branches { - if br.Name == branch { - return &RepoCommit{ - ID: br.Commit.ID, - Title: br.Commit.Title, - AuthorName: br.Commit.AuthorName, - CreatedAt: br.Commit.CreatedAt, - Message: br.Commit.Message, - }, nil - } - } } return nil, errors.New(ch.Type + "is not supported yet") } func QueryByTag(id int, owner, name, tag string, log *zap.SugaredLogger) (*RepoCommit, error) { - opt := &codehost.Option{ - CodeHostID: id, - } - ch, err := codehost.GetCodeHostInfo(opt) + ch, err := systemconfig.New().GetCodeHost(id) if err != nil { return nil, err } @@ -137,7 +126,7 @@ func QueryByTag(id int, owner, name, tag string, log *zap.SugaredLogger) (*RepoC Message: br.Commit.Message, }, nil } else if ch.Type == setting.SourceFromGerrit { - cli := gerrit.NewClient(ch.Address, ch.AccessToken) + cli := gerrit.NewClient(ch.Address, ch.AccessToken, config.ProxyHTTPSAddr(), ch.EnableProxy) commit, err := cli.GetCommitByTag(name, tag) if err != nil { return nil, err @@ -152,23 +141,6 @@ func QueryByTag(id int, owner, name, tag string, log *zap.SugaredLogger) (*RepoC CreatedAt: &commitDate, Message: commit.Message, }, nil - } else if ch.Type == setting.SourceFromIlyshin { - client := ilyshin.NewClient(ch.Address, ch.AccessToken) - tags, err := client.ListTags(owner, name, log) - if err != nil { - return nil, err - } - for _, tagInfo := range tags { - if tagInfo.Name == tag { - return &RepoCommit{ - ID: tagInfo.Commit.ID, - Title: tagInfo.Commit.Title, - AuthorName: tagInfo.Commit.AuthorName, - CreatedAt: tagInfo.Commit.CreatedAt, - Message: tagInfo.Commit.Message, - }, nil - } - } } return nil, errors.New(ch.Type + "is not supported yet") @@ -185,10 +157,7 @@ type PRCommit struct { func GetLatestPrCommit(codehostID, pr int, namespace, projectName string, log *zap.SugaredLogger) (*PRCommit, error) { projectID := fmt.Sprintf("%s/%s", namespace, projectName) - opt := &codehost.Option{ - CodeHostID: codehostID, - } - ch, err := codehost.GetCodeHostInfo(opt) + ch, err := systemconfig.New().GetCodeHost(codehostID) if err != nil { return nil, err } @@ -200,7 +169,7 @@ func GetLatestPrCommit(codehostID, pr int, namespace, projectName string, log *z } if ch.Type == gerrit.CodehostTypeGerrit { - cli := gerrit.NewClient(ch.Address, ch.AccessToken) + cli := gerrit.NewClient(ch.Address, ch.AccessToken, config.ProxyHTTPSAddr(), ch.EnableProxy) change, err := cli.GetCurrentVersionByChangeID(projectName, pr) if err != nil { return nil, err @@ -219,18 +188,6 @@ func GetLatestPrCommit(codehostID, pr int, namespace, projectName string, log *z CreatedAt: &tm, CheckoutRef: change.Revisions[change.CurrentRevision].Ref, }, nil - } else if ch.Type == setting.SourceFromIlyshin { - client := ilyshin.NewClient(ch.Address, ch.AccessToken) - commit, err := client.GetLatestPRCommitList(projectID, pr, log) - if err != nil { - return nil, err - } - return &PRCommit{ - ID: commit.ID, - Title: commit.Title, - AuthorName: commit.AuthorName, - CreatedAt: commit.CreatedAt, - }, nil } return GetLatestPRCommitList(cli, projectID, pr) } diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/job/job.go b/pkg/microservice/aslan/core/workflow/service/workflow/job/job.go new file mode 100644 index 0000000000000000000000000000000000000000..2178dfcb985a82732942cb83010986c4f8469c31 --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/service/workflow/job/job.go @@ -0,0 +1,114 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package job + +import ( + "fmt" + "strconv" + "strings" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/types" +) + +type JobCtl interface { + Instantiate() error + SetPreset() error + ToJobs(taskID int64) ([]*commonmodels.JobTask, error) +} + +func initJobCtl(job *commonmodels.Job, workflow *commonmodels.WorkflowV4) (JobCtl, error) { + var resp JobCtl + switch job.JobType { + case config.JobZadigBuild: + resp = &BuildJob{job: job, workflow: workflow} + case config.JobZadigDeploy: + resp = &DeployJob{job: job, workflow: workflow} + case config.JobPlugin: + resp = &PluginJob{job: job, workflow: workflow} + case config.JobFreestyle: + resp = &FreeStyleJob{job: job, workflow: workflow} + default: + return resp, fmt.Errorf("job type not found %s", job.JobType) + } + return resp, nil +} + +func Instantiate(job *commonmodels.Job, workflow *commonmodels.WorkflowV4) error { + ctl, err := initJobCtl(job, workflow) + if err != nil { + return err + } + return ctl.Instantiate() +} + +func SetPreset(job *commonmodels.Job, workflow *commonmodels.WorkflowV4) error { + jobCtl, err := initJobCtl(job, workflow) + if err != nil { + return err + } + return jobCtl.SetPreset() +} + +func ToJobs(job *commonmodels.Job, workflow *commonmodels.WorkflowV4, taskID int64) ([]*commonmodels.JobTask, error) { + jobCtl, err := initJobCtl(job, workflow) + if err != nil { + return []*commonmodels.JobTask{}, err + } + return jobCtl.ToJobs(taskID) +} + +// use service name and service module hash to generate job name +func jobNameFormat(jobName string) string { + if len(jobName) > 63 { + jobName = jobName[:63] + } + jobName = strings.Trim(jobName, "-") + return jobName +} + +func getReposVariables(repos []*types.Repository) []*commonmodels.KeyVal { + ret := make([]*commonmodels.KeyVal, 0) + for index, repo := range repos { + + repoNameIndex := fmt.Sprintf("REPONAME_%d", index) + ret = append(ret, &commonmodels.KeyVal{Key: fmt.Sprintf(repoNameIndex), Value: repo.RepoName, IsCredential: false}) + + repoName := strings.Replace(repo.RepoName, "-", "_", -1) + + repoIndex := fmt.Sprintf("REPO_%d", index) + ret = append(ret, &commonmodels.KeyVal{Key: fmt.Sprintf(repoIndex), Value: repoName, IsCredential: false}) + + if len(repo.Branch) > 0 { + ret = append(ret, &commonmodels.KeyVal{Key: fmt.Sprintf("%s_BRANCH", repoName), Value: repo.Branch, IsCredential: false}) + } + + if len(repo.Tag) > 0 { + ret = append(ret, &commonmodels.KeyVal{Key: fmt.Sprintf("%s_TAG", repoName), Value: repo.Tag, IsCredential: false}) + } + + if repo.PR > 0 { + ret = append(ret, &commonmodels.KeyVal{Key: fmt.Sprintf("%s_PR", repoName), Value: strconv.Itoa(repo.PR), IsCredential: false}) + } + + if len(repo.CommitID) > 0 { + ret = append(ret, &commonmodels.KeyVal{Key: fmt.Sprintf("%s_COMMIT_ID", repoName), Value: repo.CommitID, IsCredential: false}) + } + } + return ret +} diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/job/job_build.go b/pkg/microservice/aslan/core/workflow/service/workflow/job/job_build.go new file mode 100644 index 0000000000000000000000000000000000000000..ada0b94c1eff036328daa53afbff8ebfe4977704 --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/service/workflow/job/job_build.go @@ -0,0 +1,432 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package job + +import ( + "fmt" + "os" + "path" + "strings" + + configbase "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + templ "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/template" + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types" + "github.com/koderover/zadig/pkg/types/step" +) + +type BuildJob struct { + job *commonmodels.Job + workflow *commonmodels.WorkflowV4 + spec *commonmodels.ZadigBuildJobSpec +} + +func (j *BuildJob) Instantiate() error { + j.spec = &commonmodels.ZadigBuildJobSpec{} + if err := commonmodels.IToiYaml(j.job.Spec, j.spec); err != nil { + return err + } + j.job.Spec = j.spec + return nil +} + +func (j *BuildJob) SetPreset() error { + j.spec = &commonmodels.ZadigBuildJobSpec{} + if err := commonmodels.IToi(j.job.Spec, j.spec); err != nil { + return err + } + j.job.Spec = j.spec + + for _, build := range j.spec.ServiceAndBuilds { + buildInfo, err := commonrepo.NewBuildColl().Find(&commonrepo.BuildFindOption{Name: build.BuildName}) + if err != nil { + return err + } + if err := fillBuildDetail(buildInfo, build.ServiceName, build.ServiceModule); err != nil { + return err + } + for _, target := range buildInfo.Targets { + if target.ServiceName == build.ServiceName && target.ServiceModule == build.ServiceModule { + build.Repos = mergeRepoBranches(buildInfo.Repos, build.Repos) + build.KeyVals = renderKeyVals(build.KeyVals, buildInfo.PreBuild.Envs) + break + } + } + } + j.job.Spec = j.spec + return nil +} + +func (j *BuildJob) ToJobs(taskID int64) ([]*commonmodels.JobTask, error) { + logger := log.SugaredLogger() + resp := []*commonmodels.JobTask{} + + j.spec = &commonmodels.ZadigBuildJobSpec{} + if err := commonmodels.IToi(j.job.Spec, j.spec); err != nil { + return resp, err + } + j.job.Spec = j.spec + + registry, err := commonrepo.NewRegistryNamespaceColl().Find(&commonrepo.FindRegOps{ID: j.spec.DockerRegistryID}) + if err != nil { + return resp, err + } + defaultS3, err := commonrepo.NewS3StorageColl().FindDefault() + if err != nil { + return resp, err + } + + for _, build := range j.spec.ServiceAndBuilds { + imageTag := commonservice.ReleaseCandidate(build.Repos, taskID, j.workflow.Project, build.ServiceName, "", build.ServiceModule, "image") + + if len(registry.Namespace) > 0 { + build.Image = fmt.Sprintf("%s/%s/%s", registry.RegAddr, registry.Namespace, imageTag) + } else { + build.Image = fmt.Sprintf("%s/%s", registry.RegAddr, imageTag) + } + + build.Image = strings.TrimPrefix(build.Image, "http://") + build.Image = strings.TrimPrefix(build.Image, "https://") + + build.Package = fmt.Sprintf("%s.tar.gz", commonservice.ReleaseCandidate(build.Repos, taskID, j.workflow.Project, build.ServiceName, "", build.ServiceModule, "tar")) + + jobTask := &commonmodels.JobTask{ + Name: jobNameFormat(build.ServiceName + "-" + build.ServiceModule + "-" + j.job.Name), + JobType: string(config.JobZadigBuild), + } + buildInfo, err := commonrepo.NewBuildColl().Find(&commonrepo.BuildFindOption{Name: build.BuildName}) + if err != nil { + return resp, err + } + if err := fillBuildDetail(buildInfo, build.ServiceName, build.ServiceModule); err != nil { + return resp, err + } + basicImage, err := commonrepo.NewBasicImageColl().Find(buildInfo.PreBuild.ImageID) + if err != nil { + return resp, err + } + jobTask.Properties = commonmodels.JobProperties{ + Timeout: int64(buildInfo.Timeout), + ResourceRequest: buildInfo.PreBuild.ResReq, + ResReqSpec: buildInfo.PreBuild.ResReqSpec, + Envs: renderKeyVals(build.KeyVals, buildInfo.PreBuild.Envs), + ClusterID: buildInfo.PreBuild.ClusterID, + BuildOS: basicImage.Value, + ImageFrom: buildInfo.PreBuild.ImageFrom, + } + clusterInfo, err := commonrepo.NewK8SClusterColl().Get(buildInfo.PreBuild.ClusterID) + if err != nil { + return resp, err + } + + if clusterInfo.Cache.MediumType == "" { + jobTask.Properties.CacheEnable = false + } else { + jobTask.Properties.Cache = clusterInfo.Cache + jobTask.Properties.CacheEnable = buildInfo.CacheEnable + jobTask.Properties.CacheDirType = buildInfo.CacheDirType + jobTask.Properties.CacheUserDir = buildInfo.CacheUserDir + } + jobTask.Properties.Envs = append(jobTask.Properties.Envs, getBuildJobVariables(build, taskID, j.workflow.Project, j.workflow.Name, j.spec.DockerRegistryID)...) + + if jobTask.Properties.CacheEnable && jobTask.Properties.Cache.MediumType == types.NFSMedium { + jobTask.Properties.CacheUserDir = renderEnv(jobTask.Properties.CacheUserDir, jobTask.Properties.Envs) + jobTask.Properties.Cache.NFSProperties.Subpath = renderEnv(jobTask.Properties.Cache.NFSProperties.Subpath, jobTask.Properties.Envs) + } + + // init tools install step + tools := []*step.Tool{} + for _, tool := range buildInfo.PreBuild.Installs { + tools = append(tools, &step.Tool{ + Name: tool.Name, + Version: tool.Version, + }) + } + toolInstallStep := &commonmodels.StepTask{ + Name: fmt.Sprintf("%s-%s", build.ServiceName, "tool-install"), + JobName: jobTask.Name, + StepType: config.StepTools, + Spec: step.StepToolInstallSpec{Installs: tools}, + } + jobTask.Steps = append(jobTask.Steps, toolInstallStep) + // init git clone step + gitStep := &commonmodels.StepTask{ + Name: build.ServiceName + "-git", + JobName: jobTask.Name, + StepType: config.StepGit, + Spec: step.StepGitSpec{Repos: renderRepos(build.Repos, buildInfo.Repos)}, + } + jobTask.Steps = append(jobTask.Steps, gitStep) + + // init shell step + dockerLoginCmd := `docker login -u "$DOCKER_REGISTRY_AK" -p "$DOCKER_REGISTRY_SK" "$DOCKER_REGISTRY_HOST" &> /dev/null` + scripts := append([]string{dockerLoginCmd}, strings.Split(replaceWrapLine(buildInfo.Scripts), "\n")...) + shellStep := &commonmodels.StepTask{ + Name: build.ServiceName + "-shell", + JobName: jobTask.Name, + StepType: config.StepShell, + Spec: &step.StepShellSpec{ + Scripts: scripts, + }, + } + jobTask.Steps = append(jobTask.Steps, shellStep) + + // init docker build step + if buildInfo.PostBuild.DockerBuild != nil { + dockefileContent := "" + if buildInfo.PostBuild.DockerBuild.TemplateID != "" { + if dockerfileDetail, err := templ.GetDockerfileTemplateDetail(buildInfo.PostBuild.DockerBuild.TemplateID, logger); err == nil { + dockefileContent = dockerfileDetail.Content + } + } + + dockerBuildStep := &commonmodels.StepTask{ + Name: build.ServiceName + "-docker-build", + JobName: jobTask.Name, + StepType: config.StepDockerBuild, + Spec: step.StepDockerBuildSpec{ + Source: buildInfo.PostBuild.DockerBuild.Source, + WorkDir: buildInfo.PostBuild.DockerBuild.WorkDir, + DockerFile: buildInfo.PostBuild.DockerBuild.DockerFile, + ImageName: build.Image, + ImageReleaseTag: imageTag, + BuildArgs: buildInfo.PostBuild.DockerBuild.BuildArgs, + DockerTemplateContent: dockefileContent, + DockerRegistry: &step.DockerRegistry{ + DockerRegistryID: j.spec.DockerRegistryID, + }, + }, + } + jobTask.Steps = append(jobTask.Steps, dockerBuildStep) + } + + // init archive step + if buildInfo.PostBuild.FileArchive != nil && buildInfo.PostBuild.FileArchive.FileLocation != "" { + uploads := []*step.Upload{ + { + FilePath: path.Join(buildInfo.PostBuild.FileArchive.FileLocation, build.Package), + DestinationPath: path.Join(j.workflow.Name, fmt.Sprint(taskID), "archive"), + }, + } + archiveStep := &commonmodels.StepTask{ + Name: build.ServiceName + "-archive", + JobName: jobTask.Name, + StepType: config.StepArchive, + Spec: step.StepArchiveSpec{ + UploadDetail: uploads, + S3: modelS3toS3(defaultS3), + }, + } + jobTask.Steps = append(jobTask.Steps, archiveStep) + } + + // init object storage step + if buildInfo.PostBuild.ObjectStorageUpload != nil && buildInfo.PostBuild.ObjectStorageUpload.Enabled { + modelS3, err := commonrepo.NewS3StorageColl().Find(buildInfo.PostBuild.ObjectStorageUpload.ObjectStorageID) + if err != nil { + return resp, err + } + s3 := modelS3toS3(modelS3) + s3.Subfolder = "" + uploads := []*step.Upload{} + archiveStep := &commonmodels.StepTask{ + Name: build.ServiceName + "-object-storage", + JobName: jobTask.Name, + StepType: config.StepArchive, + Spec: step.StepArchiveSpec{ + UploadDetail: uploads, + ObjectStorageID: buildInfo.PostBuild.ObjectStorageUpload.ObjectStorageID, + S3: s3, + }, + } + for _, detail := range buildInfo.PostBuild.ObjectStorageUpload.UploadDetail { + uploads = append(uploads, &step.Upload{ + FilePath: detail.FilePath, + DestinationPath: detail.DestinationPath, + }) + } + jobTask.Steps = append(jobTask.Steps, archiveStep) + } + + // init psot build shell step + if buildInfo.PostBuild.Scripts != "" { + scripts := append([]string{dockerLoginCmd}, strings.Split(replaceWrapLine(buildInfo.PostBuild.Scripts), "\n")...) + shellStep := &commonmodels.StepTask{ + Name: build.ServiceName + "-post-shell", + JobName: jobTask.Name, + StepType: config.StepShell, + Spec: &step.StepShellSpec{ + Scripts: scripts, + }, + } + jobTask.Steps = append(jobTask.Steps, shellStep) + } + resp = append(resp, jobTask) + } + j.job.Spec = j.spec + return resp, nil +} + +func renderKeyVals(input, origin []*commonmodels.KeyVal) []*commonmodels.KeyVal { + for i, originKV := range origin { + for _, inputKV := range input { + if originKV.Key == inputKV.Key { + origin[i] = inputKV + } + } + } + return origin +} + +func renderRepos(input, origin []*types.Repository) []*types.Repository { + for i, originRepo := range origin { + for _, inputRepo := range input { + if originRepo.RepoName == inputRepo.RepoName && originRepo.RepoOwner == inputRepo.RepoOwner { + origin[i] = inputRepo + } + } + } + return origin +} + +func replaceWrapLine(script string) string { + return strings.Replace(strings.Replace( + script, + "\r\n", + "\n", + -1, + ), "\r", "\n", -1) +} + +func getBuildJobVariables(build *commonmodels.ServiceAndBuild, taskID int64, project, workflowName, dockerRegistryID string) []*commonmodels.KeyVal { + ret := make([]*commonmodels.KeyVal, 0) + ret = append(ret, getReposVariables(build.Repos)...) + reg, err := commonrepo.NewRegistryNamespaceColl().Find(&commonrepo.FindRegOps{ID: dockerRegistryID}) + if err != nil { + log.Errorf("find docker registry by ID %s error: %v", dockerRegistryID, err) + } + ret = append(ret, &commonmodels.KeyVal{Key: "DOCKER_REGISTRY_HOST", Value: reg.RegAddr, IsCredential: false}) + ret = append(ret, &commonmodels.KeyVal{Key: "DOCKER_REGISTRY_AK", Value: reg.AccessKey, IsCredential: false}) + ret = append(ret, &commonmodels.KeyVal{Key: "DOCKER_REGISTRY_SK", Value: reg.SecretKey, IsCredential: true}) + + ret = append(ret, &commonmodels.KeyVal{Key: "TASK_ID", Value: fmt.Sprintf("%d", taskID), IsCredential: false}) + ret = append(ret, &commonmodels.KeyVal{Key: "SERVICE", Value: build.ServiceName, IsCredential: false}) + ret = append(ret, &commonmodels.KeyVal{Key: "SERVICE_MODULE", Value: build.ServiceModule, IsCredential: false}) + ret = append(ret, &commonmodels.KeyVal{Key: "PROJECT", Value: project, IsCredential: false}) + ret = append(ret, &commonmodels.KeyVal{Key: "WORKFLOW", Value: workflowName, IsCredential: false}) + ret = append(ret, &commonmodels.KeyVal{Key: "IMAGE", Value: build.Image, IsCredential: false}) + ret = append(ret, &commonmodels.KeyVal{Key: "CI", Value: "true", IsCredential: false}) + ret = append(ret, &commonmodels.KeyVal{Key: "ZADIG", Value: "true", IsCredential: false}) + buildURL := fmt.Sprintf("%s/v1/projects/detail/%s/pipelines/custom/%s/%d", configbase.SystemAddress(), project, workflowName, taskID) + ret = append(ret, &commonmodels.KeyVal{Key: "BUILD_URL", Value: buildURL, IsCredential: false}) + ret = append(ret, &commonmodels.KeyVal{Key: "PKG_FILE", Value: build.Package, IsCredential: false}) + return ret +} + +func modelS3toS3(modelS3 *commonmodels.S3Storage) *step.S3 { + resp := &step.S3{ + Ak: modelS3.Ak, + Sk: modelS3.Sk, + Endpoint: modelS3.Endpoint, + Bucket: modelS3.Bucket, + Subfolder: modelS3.Subfolder, + Insecure: modelS3.Insecure, + Provider: modelS3.Provider, + } + if modelS3.Insecure { + resp.Protocol = "http" + } + return resp +} + +func fillBuildDetail(moduleBuild *commonmodels.Build, serviceName, serviceModule string) error { + if moduleBuild.TemplateID == "" { + return nil + } + buildTemplate, err := commonrepo.NewBuildTemplateColl().Find(&commonrepo.BuildTemplateQueryOption{ + ID: moduleBuild.TemplateID, + }) + if err != nil { + return fmt.Errorf("failed to find build template with id: %s, err: %s", moduleBuild.TemplateID, err) + } + + moduleBuild.Timeout = buildTemplate.Timeout + moduleBuild.PreBuild = buildTemplate.PreBuild + moduleBuild.JenkinsBuild = buildTemplate.JenkinsBuild + moduleBuild.Scripts = buildTemplate.Scripts + moduleBuild.PostBuild = buildTemplate.PostBuild + moduleBuild.SSHs = buildTemplate.SSHs + moduleBuild.PMDeployScripts = buildTemplate.PMDeployScripts + moduleBuild.CacheEnable = buildTemplate.CacheEnable + moduleBuild.CacheDirType = buildTemplate.CacheDirType + moduleBuild.CacheUserDir = buildTemplate.CacheUserDir + moduleBuild.AdvancedSettingsModified = buildTemplate.AdvancedSettingsModified + + // repos are configured by service modules + for _, serviceConfig := range moduleBuild.Targets { + if serviceConfig.ServiceName == serviceName && serviceConfig.ServiceModule == serviceModule { + moduleBuild.Repos = serviceConfig.Repos + if moduleBuild.PreBuild == nil { + moduleBuild.PreBuild = &commonmodels.PreBuild{} + } + moduleBuild.PreBuild.Envs = commonservice.MergeBuildEnvs(moduleBuild.PreBuild.Envs, serviceConfig.Envs) + break + } + } + return nil +} + +func renderEnv(data string, kvs []*commonmodels.KeyVal) string { + mapper := func(data string) string { + for _, envar := range kvs { + if data != envar.Key { + continue + } + + return envar.Value + } + + return fmt.Sprintf("$%s", data) + } + return os.Expand(data, mapper) +} + +func mergeRepoBranches(templateRepos []*types.Repository, customRepos []*types.Repository) []*types.Repository { + customRepoMap := make(map[string]*types.Repository) + for _, repo := range customRepos { + if repo.RepoNamespace == "" { + repo.RepoNamespace = repo.RepoOwner + } + repoKey := strings.Join([]string{repo.Source, repo.RepoNamespace, repo.RepoName}, "/") + customRepoMap[repoKey] = repo + } + for _, repo := range templateRepos { + if repo.RepoNamespace == "" { + repo.RepoNamespace = repo.RepoOwner + } + repoKey := strings.Join([]string{repo.Source, repo.RepoNamespace, repo.RepoName}, "/") + // user can only set default branch in custom workflow. + if cv, ok := customRepoMap[repoKey]; ok { + repo.Branch = cv.Branch + } + } + return templateRepos +} diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/job/job_deploy.go b/pkg/microservice/aslan/core/workflow/service/workflow/job/job_deploy.go new file mode 100644 index 0000000000000000000000000000000000000000..aad667b36998be14c86d46bb433e178f2699f405 --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/service/workflow/job/job_deploy.go @@ -0,0 +1,231 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package job + +import ( + "fmt" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/types/step" + "github.com/koderover/zadig/pkg/util" +) + +type DeployJob struct { + job *commonmodels.Job + workflow *commonmodels.WorkflowV4 + spec *commonmodels.ZadigDeployJobSpec +} + +func (j *DeployJob) Instantiate() error { + j.spec = &commonmodels.ZadigDeployJobSpec{} + if err := commonmodels.IToiYaml(j.job.Spec, j.spec); err != nil { + return err + } + j.job.Spec = j.spec + return nil +} + +func (j *DeployJob) SetPreset() error { + j.spec = &commonmodels.ZadigDeployJobSpec{} + if err := commonmodels.IToi(j.job.Spec, j.spec); err != nil { + return err + } + j.job.Spec = j.spec + + project, err := templaterepo.NewProductColl().Find(j.workflow.Project) + if err != nil { + return err + } + if project.ProductFeature != nil { + j.spec.DeployType = project.ProductFeature.DeployType + } + + services, err := commonrepo.NewServiceColl().ListMaxRevisionsByProduct(j.workflow.Project) + if err != nil { + return err + } + + if j.spec.Source != config.SourceRuntime { + return nil + } + + for _, service := range services { + for _, container := range service.Containers { + j.spec.ServiceAndImages = append(j.spec.ServiceAndImages, &commonmodels.ServiceAndImage{ServiceName: service.ServiceName, ServiceModule: container.Name}) + } + } + j.job.Spec = j.spec + return nil +} + +func (j *DeployJob) ToJobs(taskID int64) ([]*commonmodels.JobTask, error) { + resp := []*commonmodels.JobTask{} + + j.spec = &commonmodels.ZadigDeployJobSpec{} + if err := commonmodels.IToi(j.job.Spec, j.spec); err != nil { + return resp, err + } + j.job.Spec = j.spec + + product, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{Name: j.workflow.Project, EnvName: j.spec.Env}) + if err != nil { + return resp, fmt.Errorf("env %s not exists", j.spec.Env) + } + + project, err := templaterepo.NewProductColl().Find(j.workflow.Project) + if err != nil { + return resp, err + } + + productServiceMap := product.GetServiceMap() + + if project.ProductFeature != nil && project.ProductFeature.CreateEnvType == setting.SourceFromExternal { + productServices, err := commonrepo.NewServiceColl().ListExternalWorkloadsBy(j.workflow.Project, j.spec.Env) + if err != nil { + return resp, err + } + for _, service := range productServices { + productServiceMap[service.ServiceName] = &commonmodels.ProductService{ + ServiceName: service.ServiceName, + Containers: service.Containers, + } + } + servicesInExternalEnv, _ := commonrepo.NewServicesInExternalEnvColl().List(&commonrepo.ServicesInExternalEnvArgs{ + ProductName: j.workflow.Project, + EnvName: j.spec.Env, + }) + for _, service := range servicesInExternalEnv { + productServiceMap[service.ServiceName] = &commonmodels.ProductService{ + ServiceName: service.ServiceName, + } + } + } + + // get deploy info from previous build job + if j.spec.Source == config.SourceFromJob { + for _, stage := range j.workflow.Stages { + for _, job := range stage.Jobs { + if job.JobType != config.JobZadigBuild || job.Name != j.spec.JobName { + continue + } + buildSpec := &commonmodels.ZadigBuildJobSpec{} + if err := commonmodels.IToi(job.Spec, buildSpec); err != nil { + return resp, err + } + for _, build := range buildSpec.ServiceAndBuilds { + j.spec.ServiceAndImages = append(j.spec.ServiceAndImages, &commonmodels.ServiceAndImage{ + ServiceName: build.ServiceName, + ServiceModule: build.ServiceModule, + Image: build.Image, + }) + } + } + } + } + if j.spec.DeployType == setting.K8SDeployType { + for _, deploy := range j.spec.ServiceAndImages { + if err := checkServiceExsistsInEnv(productServiceMap, deploy.ServiceName, j.spec.Env); err != nil { + return resp, err + } + jobTask := &commonmodels.JobTask{ + Name: jobNameFormat(deploy.ServiceName + "-" + deploy.ServiceModule + "-" + j.job.Name), + JobType: string(config.JobZadigDeploy), + } + deployStep := &commonmodels.StepTask{ + Name: deploy.ServiceName + "-deploy", + JobName: jobTask.Name, + StepType: config.StepDeploy, + Spec: step.StepDeploySpec{ + Env: j.spec.Env, + ServiceName: deploy.ServiceName, + ServiceType: setting.K8SDeployType, + ServiceModule: deploy.ServiceModule, + ClusterID: product.ClusterID, + Image: deploy.Image, + }, + } + jobTask.Steps = append(jobTask.Steps, deployStep) + resp = append(resp, jobTask) + } + } + if j.spec.DeployType == setting.HelmDeployType { + deployServiceMap := map[string][]*commonmodels.ServiceAndImage{} + for _, deploy := range j.spec.ServiceAndImages { + deployServiceMap[deploy.ServiceName] = append(deployServiceMap[deploy.ServiceName], deploy) + } + for serviceName, deploys := range deployServiceMap { + var serviceRevision int64 + if pSvc, ok := productServiceMap[serviceName]; ok { + serviceRevision = pSvc.Revision + } + + revisionSvc, err := commonrepo.NewServiceColl().Find(&commonrepo.ServiceFindOption{ + ServiceName: serviceName, + Revision: serviceRevision, + ProductName: product.ProductName, + }) + if err != nil { + return nil, fmt.Errorf("failed to find service: %s with revision: %d, err: %s", serviceName, serviceRevision, err) + } + releaseName := util.GeneReleaseName(revisionSvc.GetReleaseNaming(), product.ProductName, product.Namespace, product.EnvName, serviceName) + + jobTask := &commonmodels.JobTask{ + Name: jobNameFormat(serviceName + "-" + j.job.Name), + JobType: string(config.JobZadigDeploy), + } + deployStep := &commonmodels.StepTask{ + Name: serviceName + "-deploy", + JobName: jobTask.Name, + StepType: config.StepHelmDeploy, + } + helmDeploySpec := step.StepHelmDeploySpec{ + Env: j.spec.Env, + ServiceName: serviceName, + ServiceType: setting.HelmDeployType, + ClusterID: product.ClusterID, + ReleaseName: releaseName, + } + for _, deploy := range deploys { + if err := checkServiceExsistsInEnv(productServiceMap, serviceName, j.spec.Env); err != nil { + return resp, err + } + helmDeploySpec.ImageAndModules = append(helmDeploySpec.ImageAndModules, &step.ImageAndServiceModule{ + ServiceModule: deploy.ServiceModule, + Image: deploy.Image, + }) + } + deployStep.Spec = helmDeploySpec + + jobTask.Steps = append(jobTask.Steps, deployStep) + resp = append(resp, jobTask) + } + } + + j.job.Spec = j.spec + return resp, nil +} + +func checkServiceExsistsInEnv(serviceMap map[string]*commonmodels.ProductService, serviceName, env string) error { + if _, ok := serviceMap[serviceName]; !ok { + return fmt.Errorf("service %s not exists in env %s", serviceName, env) + } + return nil +} diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/job/job_freestyle.go b/pkg/microservice/aslan/core/workflow/service/workflow/job/job_freestyle.go new file mode 100644 index 0000000000000000000000000000000000000000..255da89404af8b08ca9b4d1e45ceda5d00a4ed3e --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/service/workflow/job/job_freestyle.go @@ -0,0 +1,141 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package job + +import ( + "fmt" + + configbase "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types" + steptypes "github.com/koderover/zadig/pkg/types/step" +) + +type FreeStyleJob struct { + job *commonmodels.Job + workflow *commonmodels.WorkflowV4 + spec *commonmodels.FreestyleJobSpec +} + +func (j *FreeStyleJob) Instantiate() error { + j.spec = &commonmodels.FreestyleJobSpec{} + if err := commonmodels.IToiYaml(j.job.Spec, j.spec); err != nil { + return err + } + for _, step := range j.spec.Steps { + switch step.StepType { + case config.StepTools: + stepSpec := &steptypes.StepToolInstallSpec{} + if err := commonmodels.IToiYaml(step.Spec, stepSpec); err != nil { + return err + } + step.Spec = stepSpec + case config.StepGit: + stepSpec := &steptypes.StepGitSpec{} + if err := commonmodels.IToiYaml(step.Spec, stepSpec); err != nil { + return err + } + step.Spec = stepSpec + case config.StepShell: + stepSpec := &steptypes.StepShellSpec{} + if err := commonmodels.IToiYaml(step.Spec, stepSpec); err != nil { + return err + } + step.Spec = stepSpec + case config.StepArchive: + stepSpec := &steptypes.StepArchiveSpec{} + if err := commonmodels.IToiYaml(step.Spec, stepSpec); err != nil { + return err + } + step.Spec = stepSpec + default: + return fmt.Errorf("freestyle job step type %s not supported", step.StepType) + } + + } + j.job.Spec = j.spec + return nil +} + +func (j *FreeStyleJob) SetPreset() error { + j.spec = &commonmodels.FreestyleJobSpec{} + if err := commonmodels.IToi(j.job.Spec, j.spec); err != nil { + return err + } + j.job.Spec = j.spec + return nil +} + +func (j *FreeStyleJob) ToJobs(taskID int64) ([]*commonmodels.JobTask, error) { + resp := []*commonmodels.JobTask{} + j.spec = &commonmodels.FreestyleJobSpec{} + if err := commonmodels.IToi(j.job.Spec, j.spec); err != nil { + return resp, err + } + j.job.Spec = j.spec + jobTask := &commonmodels.JobTask{ + Name: j.job.Name, + JobType: string(config.JobFreestyle), + Properties: *j.spec.Properties, + Steps: stepsToStepTasks(j.spec.Steps), + } + basicImage, err := commonrepo.NewBasicImageColl().Find(jobTask.Properties.ImageID) + if err != nil { + return resp, err + } + jobTask.Properties.BuildOS = basicImage.Value + jobTask.Properties.Envs = append(jobTask.Properties.Envs, getfreestyleJobVariables(jobTask.Steps, taskID, j.workflow.Project, j.workflow.Name)...) + return []*commonmodels.JobTask{jobTask}, nil +} + +func stepsToStepTasks(step []*commonmodels.Step) []*commonmodels.StepTask { + resp := []*commonmodels.StepTask{} + for _, step := range step { + stepTask := &commonmodels.StepTask{ + Name: step.Name, + StepType: step.StepType, + Spec: step.Spec, + } + resp = append(resp, stepTask) + } + return resp +} + +func getfreestyleJobVariables(steps []*commonmodels.StepTask, taskID int64, project, workflowName string) []*commonmodels.KeyVal { + ret := []*commonmodels.KeyVal{} + repos := []*types.Repository{} + for _, step := range steps { + if step.StepType != config.StepGit { + continue + } + stepSpec := &steptypes.StepGitSpec{} + if err := commonmodels.IToi(step.Spec, stepSpec); err != nil { + log.Errorf("failed to convert step spec error: %v", err) + continue + } + repos = append(repos, stepSpec.Repos...) + } + ret = append(ret, getReposVariables(repos)...) + + ret = append(ret, &commonmodels.KeyVal{Key: "TASK_ID", Value: fmt.Sprintf("%d", taskID), IsCredential: false}) + buildURL := fmt.Sprintf("%s/v1/projects/detail/%s/pipelines/custom/%s/%d", configbase.SystemAddress(), project, workflowName, taskID) + ret = append(ret, &commonmodels.KeyVal{Key: "BUILD_URL", Value: buildURL, IsCredential: false}) + return ret +} diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/job/job_plugin.go b/pkg/microservice/aslan/core/workflow/service/workflow/job/job_plugin.go new file mode 100644 index 0000000000000000000000000000000000000000..1736b86feef3b4a5b01742f2b9dc3fd2bbbe2060 --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/service/workflow/job/job_plugin.go @@ -0,0 +1,88 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package job + +import ( + "strings" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" +) + +type PluginJob struct { + job *commonmodels.Job + workflow *commonmodels.WorkflowV4 + spec *commonmodels.PluginJobSpec +} + +func (j *PluginJob) Instantiate() error { + j.spec = &commonmodels.PluginJobSpec{} + if err := commonmodels.IToiYaml(j.job.Spec, j.spec); err != nil { + return err + } + j.job.Spec = j.spec + return nil +} + +func (j *PluginJob) SetPreset() error { + j.spec = &commonmodels.PluginJobSpec{} + if err := commonmodels.IToi(j.job.Spec, j.spec); err != nil { + return err + } + j.job.Spec = j.spec + return nil +} + +func (j *PluginJob) ToJobs(taskID int64) ([]*commonmodels.JobTask, error) { + resp := []*commonmodels.JobTask{} + j.spec = &commonmodels.PluginJobSpec{} + if err := commonmodels.IToi(j.job.Spec, j.spec); err != nil { + return resp, err + } + j.job.Spec = j.spec + jobTask := &commonmodels.JobTask{ + Name: j.job.Name, + JobType: string(config.JobPlugin), + Properties: *j.spec.Properties, + Plugin: j.spec.Plugin, + } + for _, output := range j.spec.Plugin.Outputs { + jobTask.Outputs = append(jobTask.Outputs, &commonmodels.Output{Name: output.Name, Description: output.Description}) + } + jobTask.Plugin = renderPlugin(jobTask.Plugin, j.spec.Plugin.Inputs) + return []*commonmodels.JobTask{jobTask}, nil +} + +func renderPlugin(plugin *commonmodels.PluginTemplate, inputs []*commonmodels.Params) *commonmodels.PluginTemplate { + for _, env := range plugin.Envs { + env.Value = renderString(env.Value, inputs) + } + for i, arg := range plugin.Args { + plugin.Args[i] = renderString(arg, inputs) + } + for i, cmd := range plugin.Cmds { + plugin.Cmds[i] = renderString(cmd, inputs) + } + return plugin +} + +func renderString(value string, inputs []*commonmodels.Params) string { + for _, input := range inputs { + value = strings.ReplaceAll(value, "$("+input.Name+")", input.Value) + } + return value +} diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/nsq_handlers.go b/pkg/microservice/aslan/core/workflow/service/workflow/nsq_handlers.go index 70542680bb85e6a5889a12d7dca1e8e384ac8817..626323db4ba31bdd603ec35ed294e801efae3378 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/nsq_handlers.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/nsq_handlers.go @@ -18,10 +18,8 @@ package workflow import ( "context" - "encoding/base64" "encoding/json" "encoding/xml" - "errors" "fmt" "io/ioutil" "net/http" @@ -33,6 +31,7 @@ import ( dockerCli "github.com/docker/docker/client" "github.com/docker/go-connections/sockets" "github.com/nsqio/go-nsq" + "github.com/pkg/errors" "go.uber.org/zap" "github.com/koderover/zadig/pkg/microservice/aslan/config" @@ -46,10 +45,11 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/registry" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/poetry" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" "github.com/koderover/zadig/pkg/tool/git/gitlab" "github.com/koderover/zadig/pkg/tool/log" s3tool "github.com/koderover/zadig/pkg/tool/s3" + "github.com/koderover/zadig/pkg/types" "github.com/koderover/zadig/pkg/util" ) @@ -64,12 +64,11 @@ type TaskAckHandler struct { deliveryArtifactColl *commonrepo.DeliveryArtifactColl deliveryActivityColl *commonrepo.DeliveryActivityColl TestTaskStatColl *commonrepo.TestTaskStatColl - PoetryClient *poetry.Client messages chan *nsq.Message log *zap.SugaredLogger } -func NewTaskAckHandler(poetryServer, poetryRootKey string, maxInFlight int, log *zap.SugaredLogger) *TaskAckHandler { +func NewTaskAckHandler(maxInFlight int, log *zap.SugaredLogger) *TaskAckHandler { return &TaskAckHandler{ queue: NewPipelineQueue(log), ptColl: commonrepo.NewTaskColl(), @@ -79,7 +78,6 @@ func NewTaskAckHandler(poetryServer, poetryRootKey string, maxInFlight int, log deliveryArtifactColl: commonrepo.NewDeliveryArtifactColl(), deliveryActivityColl: commonrepo.NewDeliveryActivityColl(), TestTaskStatColl: commonrepo.NewTestTaskStatColl(), - PoetryClient: poetry.New(poetryServer, poetryRootKey), messages: make(chan *nsq.Message, maxInFlight*10), log: log, } @@ -198,7 +196,8 @@ func (h *TaskAckHandler) handle(message *nsq.Message) error { for _, deploy := range deploys { if deploy.Enabled && !pt.ResetImage { - if err := h.updateProductImageByNs(deploy.Namespace, deploy.ProductName, deploy.ServiceName, deploy.ContainerName, deploy.Image); err != nil { + containerName := strings.TrimSuffix(deploy.ContainerName, "_"+deploy.ServiceName) + if err := h.updateProductImageByNs(deploy.Namespace, deploy.ProductName, deploy.ServiceName, containerName, deploy.Image); err != nil { h.log.Errorf("updateProductImage %v error: %v", deploy, err) continue } else { @@ -207,14 +206,65 @@ func (h *TaskAckHandler) handle(message *nsq.Message) error { } } - // 更新历史pipeline状态（默认留下前一百个） - if err = h.ptColl.ArchiveHistoryPipelineTask(pt.PipelineName, pt.Type, 100); err != nil { + // update history tasks status to archive + result, err := commonrepo.NewStrategyColl().GetByTarget(commonmodels.WorkflowTaskRetention) + if err != nil { + h.log.Errorf("get workflow task retention strategy error: %s", err) + result = commonmodels.DefaultWorkflowTaskRetention + } + if err = h.ptColl.ArchiveHistoryPipelineTask(pt.PipelineName, pt.Type, result.Retention.MaxItems, result.Retention.MaxDays); err != nil { h.log.Errorf("ArchiveHistoryPipelineTask error: %v", err) } return nil } +// get docker file content from codehost +// TODO need to support codehub and gerrit +func getRawFileContent(codehostID int, repo, owner, branch, filePath string) ([]byte, error) { + ch, err := systemconfig.New().GetCodeHost(codehostID) + if err != nil { + return nil, errors.Wrapf(err, "Failed to get codeHost info %d", codehostID) + } + switch ch.Type { + case setting.SourceFromGitlab: + cli, err := gitlab.NewClient(ch.ID, ch.Address, ch.AccessToken, config.ProxyHTTPSAddr(), ch.EnableProxy) + if err != nil { + return nil, errors.Wrapf(err, "Failed to get gitlab client") + } + return cli.GetRawFile(repo, owner, branch, filePath) + case setting.SourceFromGithub: + gitClient := git.NewClient(ch.AccessToken, config.ProxyHTTPSAddr(), ch.EnableProxy) + return gitClient.GetFileContent(owner, repo, filePath, branch) + default: + return nil, fmt.Errorf("Failed to create client for codehostID: %d", codehostID) + } +} + +func (h *TaskAckHandler) getDockerfileContent(build *types.Repository, ctx *task.DockerBuildCtx) string { + switch ctx.Source { + case setting.DockerfileSourceTemplate: + return ctx.DockerTemplateContent + case setting.DockerfileSourceLocal: + path := ctx.DockerFile + pathArray := strings.Split(path, "/") + if len(pathArray[0])+1 >= len(path) { + h.log.Errorf("Failed to get dockerfile content, build context: %+v", *ctx) + return "" + } + dockerfilePath := path[len(pathArray[0])+1:] + content, err := getRawFileContent(build.CodehostID, build.RepoName, build.RepoOwner, build.Branch, dockerfilePath) + if err != nil { + h.log.Errorf("Failed to get dockerfile content, err %s", err) + return "" + } + return string(content) + default: // from local + h.log.Errorf("Failed to get dockerfile content, illegal source %s", ctx.Source) + return "" + } +} + func (h *TaskAckHandler) uploadTaskData(pt *task.Task) error { deliveryArtifacts := make([]*commonmodels.DeliveryArtifact, 0) if pt.Type == config.WorkflowType { @@ -244,8 +294,10 @@ func (h *TaskAckHandler) uploadTaskData(pt *task.Task) error { deliveryArtifact.ImageTag = buildInfo.JobCtx.FileArchiveCtx.FileName deliveryArtifact.Type = string(config.File) deliveryArtifact.PackageFileLocation = buildInfo.JobCtx.FileArchiveCtx.FileLocation - storageInfo, _ := s3.NewS3StorageFromEncryptedURI(pt.StorageURI) - deliveryArtifact.PackageStorageURI = storageInfo.Endpoint + if storageInfo, err := s3.NewS3StorageFromEncryptedURI(pt.StorageURI); err == nil { + deliveryArtifact.PackageStorageURI = storageInfo.Endpoint + "/" + storageInfo.Bucket + } + deliveryArtifactArray = append(deliveryArtifactArray, deliveryArtifact) } @@ -264,8 +316,8 @@ func (h *TaskAckHandler) uploadTaskData(pt *task.Task) error { deliveryArtifact.Type = string(config.Image) deliveryArtifact.Name = imageName deliveryArtifact.ImageTag = imageTag - //获取镜像详细信息 - imageInfo, _ := getImageInfo(imageName, imageTag, h.log) + //get image detail info + imageInfo, _ := getImageInfo(buildInfo.ProductName, buildInfo.EnvName, imageName, imageTag, h.log) if imageInfo != nil { deliveryArtifact.ImageSize = imageInfo.ImageSize deliveryArtifact.ImageDigest = imageInfo.ImageDigest @@ -290,35 +342,7 @@ func (h *TaskAckHandler) uploadTaskData(pt *task.Task) error { if buildInfo.JobCtx.DockerBuildCtx != nil { for _, build := range buildInfo.JobCtx.Builds { - path := buildInfo.JobCtx.DockerBuildCtx.DockerFile - pathArray := strings.Split(path, "/") - dockerfilePath := path[len(pathArray[0])+1:] - if strings.Contains(build.Address, "gitlab") { - cli, err := gitlab.NewClient(build.Address, build.OauthToken) - if err != nil { - h.log.Errorf("Failed to get gitlab client, err: %v", err) - continue - } - content, err := cli.GetRawFile(build.RepoOwner, build.RepoName, build.Branch, dockerfilePath) - if err != nil { - h.log.Errorf("uploadTaskData gitlab GetRawFile err:%v", err) - continue - } - deliveryArtifact.DockerFile = string(content) - } else { - gitClient := git.NewClient(build.OauthToken, config.ProxyHTTPSAddr()) - fileContent, _, _, _ := gitClient.Repositories.GetContents(context.Background(), build.RepoOwner, build.RepoName, dockerfilePath, nil) - if fileContent != nil { - dockerfileContent := *fileContent.Content - dockerfileContent = dockerfileContent[:len(dockerfileContent)-2] - content, err := base64.StdEncoding.DecodeString(dockerfileContent) - if err != nil { - h.log.Errorf("uploadTaskData github GetRawFile err:%v", err) - continue - } - deliveryArtifact.DockerFile = string(content) - } - } + deliveryArtifact.DockerFile = h.getDockerfileContent(build, buildInfo.JobCtx.DockerBuildCtx) } } deliveryArtifactArray = append(deliveryArtifactArray, deliveryArtifact) @@ -691,7 +715,7 @@ func (h *TaskAckHandler) createVersion(pt *task.Task) error { } if isDeploy { //版本交付 - return commonservice.AddDeliveryVersion(1, int(pt.TaskID), pt.ProductName, pt.PipelineName, pt, log.SugaredLogger()) + return commonservice.AddDeliveryVersion(int(pt.TaskID), pt.ProductName, pt.PipelineName, pt, log.SugaredLogger()) } } } @@ -871,7 +895,7 @@ func (h *TaskNotificationHandler) HandleMessage(message *nsq.Message) error { } func upsertWorkflowStat(args *commonmodels.WorkflowStat, log *zap.SugaredLogger) error { - if workflowStats, _ := commonrepo.NewWorkflowStatColl().FindWorkflowStat(&commonrepo.WorkflowStatArgs{Name: args.Name, Type: args.Type}); len(workflowStats) > 0 { + if workflowStats, _ := commonrepo.NewWorkflowStatColl().FindWorkflowStat(&commonrepo.WorkflowStatArgs{Names: []string{args.Name}, Type: args.Type}); len(workflowStats) > 0 { currentWorkflowStat := workflowStats[0] currentWorkflowStat.TotalDuration += args.TotalDuration currentWorkflowStat.TotalSuccess += args.TotalSuccess @@ -892,16 +916,28 @@ func upsertWorkflowStat(args *commonmodels.WorkflowStat, log *zap.SugaredLogger) return nil } -func getImageInfo(repoName, tag string, log *zap.SugaredLogger) (*commonmodels.DeliveryImage, error) { - regOps := new(commonrepo.FindRegOps) - regOps.IsDefault = true - registryInfo, err := commonrepo.NewRegistryNamespaceColl().Find(regOps) +func getImageInfo(productName, evnName, repoName, tag string, log *zap.SugaredLogger) (*commonmodels.DeliveryImage, error) { + productInfo, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: productName, + EnvName: evnName, + }) + if err != nil { + return nil, err + } + registryInfo, _, err := commonservice.FindRegistryById(productInfo.RegistryID, false, log) if err != nil { log.Errorf("RegistryNamespace.get error: %v", err) return nil, fmt.Errorf("RegistryNamespace.get error: %v", err) } - return registry.NewV2Service(registryInfo.RegProvider).GetImageInfo(registry.GetRepoImageDetailOption{ + var regService registry.Service + if registryInfo.AdvancedSetting != nil { + regService = registry.NewV2Service(registryInfo.RegProvider, registryInfo.AdvancedSetting.TLSEnabled, registryInfo.AdvancedSetting.TLSCert) + } else { + regService = registry.NewV2Service(registryInfo.RegProvider, true, "") + } + + return regService.GetImageInfo(registry.GetRepoImageDetailOption{ Endpoint: registry.Endpoint{ Addr: registryInfo.RegAddr, Ak: registryInfo.AccessKey, diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/pipeline.go b/pkg/microservice/aslan/core/workflow/service/workflow/pipeline.go index 345edc271cb166c99965f4c8100be33fc403820e..8ba33b4a1b920efa5a35c845801b5d31bfa813f7 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/pipeline.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/pipeline.go @@ -42,7 +42,7 @@ func ListPipelines(log *zap.SugaredLogger) ([]*commonmodels.Pipeline, error) { return resp, nil } -func GetPipeline(userID int, pipelineName string, log *zap.SugaredLogger) (*commonmodels.Pipeline, error) { +func GetPipeline(userID, pipelineName string, log *zap.SugaredLogger) (*commonmodels.Pipeline, error) { resp, err := commonrepo.NewPipelineColl().Find(&commonrepo.PipelineFindOption{Name: pipelineName}) if err != nil { log.Error(err) diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_controller.go b/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_controller.go index 7329538d8756a0cff77684be5cabb706f2719987..d6536ddc89a024176b9964162e3054b20efca019 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_controller.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_controller.go @@ -43,7 +43,7 @@ func SubScribeNSQ() error { // init ack consumer ackConfig := nsqservice.Config() ackConfig.MaxInFlight = 50 - ackHandler := NewTaskAckHandler(config.PoetryAPIServer(), config.PoetryAPIRootKey(), ackConfig.MaxInFlight, logger) + ackHandler := NewTaskAckHandler(ackConfig.MaxInFlight, logger) err := nsqservice.SubScribe(setting.TopicAck, "ack", 1, ackConfig, ackHandler) if err != nil { logger.Errorf("ack subscription failed, the error is: %v", err) @@ -415,12 +415,12 @@ func ParallelRunningAndQueuedTasks(currentTask *task.Task) bool { return false } //如果是产品工作流判断是否设置了并行 - if currentTask.Type == config.WorkflowType && !currentTask.WorkflowArgs.IsParallel { + if currentTask.Type == config.WorkflowType && currentTask.WorkflowArgs != nil && !currentTask.WorkflowArgs.IsParallel { return false } //优先处理webhook - if currentTask.TaskCreator == setting.WebhookTaskCreator { - return HandlerWebhookTask(currentTask) + if currentTask.TaskCreator == setting.WebhookTaskCreator && checkWebhookTaskNamespace(currentTask) { + return true } //判断如果当前的task没有部署，直接放到队列里执行 currentTaskDeployServices := sets.NewString() @@ -464,8 +464,7 @@ func ParallelRunningAndQueuedTasks(currentTask *task.Task) bool { return true } -// HandlerWebhookTask 处理webhook逻辑 -func HandlerWebhookTask(currentTask *task.Task) bool { +func checkWebhookTaskNamespace(currentTask *task.Task) bool { runningNamespaces := sets.String{} for _, t := range ListTasks() { if t.Status != config.StatusRunning && t.Status != config.StatusQueued { @@ -475,11 +474,11 @@ func HandlerWebhookTask(currentTask *task.Task) bool { continue } - if !runningNamespaces.Has(t.WorkflowArgs.Namespace) { + if t.WorkflowArgs != nil && !runningNamespaces.Has(t.WorkflowArgs.Namespace) { runningNamespaces.Insert(t.WorkflowArgs.Namespace) } } - if !runningNamespaces.Has(currentTask.WorkflowArgs.Namespace) && currentTask.WorkflowArgs.IsParallel { + if currentTask.WorkflowArgs != nil && !runningNamespaces.Has(currentTask.WorkflowArgs.Namespace) && currentTask.WorkflowArgs.IsParallel { return true } diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_status.go b/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_status.go index 3f50ddafd40b6b9b50ccfc23113d6d2e8f54d307..3d468e7e6b7d0a68890b01bd8dc3be7a3d7b71c2 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_status.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_status.go @@ -50,7 +50,7 @@ type PipelinePreview struct { TotalSuccess int `bson:"-" json:"total_success"` } -func ListPipelinesPreview(userID int, log *zap.SugaredLogger) ([]*PipelinePreview, error) { +func ListPipelinesPreview(userID string, log *zap.SugaredLogger) ([]*PipelinePreview, error) { resp := make([]*PipelinePreview, 0) pipelineList, err := commonrepo.NewPipelineColl().List(&commonrepo.PipelineListOption{}) diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_task.go b/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_task.go index cafcd8e0884ee403abceeb63f47e1e44a6909d22..ee63341a31825718853e0d970d3aab024c0f8d41 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_task.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_task.go @@ -40,7 +40,7 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/scmnotify" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/poetry" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" e "github.com/koderover/zadig/pkg/tool/errors" krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client" "github.com/koderover/zadig/pkg/tool/kube/getter" @@ -68,7 +68,9 @@ func CreatePipelineTask(args *commonmodels.TaskArgs, log *zap.SugaredLogger) (*C // 如果用户使用预定义编译配置, 则从编译模块配置中生成SubTasks if pipeline.BuildModuleVer != "" { - subTasks, err := BuildModuleToSubTasks("", pipeline.Target, "", "", nil, nil, log) + subTasks, err := BuildModuleToSubTasks(&commonmodels.BuildModuleArgs{ + Target: pipeline.Target, + }, log) if err != nil { return nil, e.ErrCreateTask.AddErr(err) } @@ -112,7 +114,7 @@ func CreatePipelineTask(args *commonmodels.TaskArgs, log *zap.SugaredLogger) (*C } if build.Registries == nil { - registries, err := commonservice.ListRegistryNamespaces(log) + registries, err := commonservice.ListRegistryNamespaces("", true, log) if err != nil { log.Errorf("ListRegistryNamespaces err:%v", err) } else { @@ -146,7 +148,7 @@ func CreatePipelineTask(args *commonmodels.TaskArgs, log *zap.SugaredLogger) (*C } if testing.Registries == nil { - registries, err := commonservice.ListRegistryNamespaces(log) + registries, err := commonservice.ListRegistryNamespaces("", true, log) if err != nil { log.Errorf("ListRegistryNamespaces err:%v", err) } else { @@ -163,7 +165,7 @@ func CreatePipelineTask(args *commonmodels.TaskArgs, log *zap.SugaredLogger) (*C } } - jiraInfo, _ := poetry.GetJiraInfo(config.PoetryAPIServer(), config.PoetryAPIRootKey()) + jiraInfo, _ := systemconfig.New().GetJiraInfo() if jiraInfo != nil { jiraTask, err := AddPipelineJiraSubTask(pipeline, log) if err != nil { @@ -196,7 +198,6 @@ func CreatePipelineTask(args *commonmodels.TaskArgs, log *zap.SugaredLogger) (*C MultiRun: pipeline.MultiRun, BuildModuleVer: pipeline.BuildModuleVer, Target: pipeline.Target, - OrgID: pipeline.OrgID, StorageURI: defaultStorageURI, } @@ -231,7 +232,7 @@ func CreatePipelineTask(args *commonmodels.TaskArgs, log *zap.SugaredLogger) (*C } } - repos, err := commonrepo.NewRegistryNamespaceColl().FindAll(&commonrepo.FindRegOps{}) + repos, err := commonservice.ListRegistryNamespaces("", false, log) if err != nil { return nil, e.ErrCreateTask.AddErr(err) } @@ -241,7 +242,10 @@ func CreatePipelineTask(args *commonmodels.TaskArgs, log *zap.SugaredLogger) (*C pt.ConfigPayload.RepoConfigs[repo.ID.Hex()] = repo } - if err := ensurePipelineTask(pt, pt.TaskArgs.Deploy.Namespace, log); err != nil { + if err := ensurePipelineTask(&task.TaskOpt{ + Task: pt, + EnvName: pt.TaskArgs.Deploy.Namespace, + }, log); err != nil { log.Errorf("Service.ensurePipelineTask failed %v %v", args, err) if err, ok := err.(*ContainerNotFound); ok { return nil, e.NewWithExtras( @@ -276,6 +280,7 @@ func CreatePipelineTask(args *commonmodels.TaskArgs, log *zap.SugaredLogger) (*C scmnotify.NewService().UpdatePipelineWebhookComment(pt, log) resp := &CreateTaskResp{ + ProjectName: args.ProductName, PipelineName: args.PipelineName, TaskID: nextTaskID, } @@ -291,49 +296,81 @@ type TaskResult struct { } // ListPipelineTasksV2Result 工作流任务分页信息 -func ListPipelineTasksV2Result(name string, typeString config.PipelineType, maxResult, startAt int, log *zap.SugaredLogger) (*TaskResult, error) { +func ListPipelineTasksV2Result(name string, typeString config.PipelineType, queryType string, filters []string, maxResult, startAt int, log *zap.SugaredLogger) (*TaskResult, error) { ret := &TaskResult{MaxResult: maxResult, StartAt: startAt} var err error - ret.Data, err = commonrepo.NewTaskColl().List(&commonrepo.ListTaskOption{PipelineName: name, Limit: maxResult, Skip: startAt, Detail: true, Type: typeString}) + var listTaskOpt *commonrepo.ListTaskOption + var countTaskOpt *commonrepo.CountTaskOption + var restp []*commonrepo.TaskPreview + if len(filters) == 0 || (len(filters) == 1 && filters[0] == "") { + queryType = "" + } + switch queryType { + case "creator": + listTaskOpt = &commonrepo.ListTaskOption{PipelineName: name, Limit: maxResult, Skip: startAt, TaskCreators: filters, ForWorkflowTaskList: true, Type: typeString} + countTaskOpt = &commonrepo.CountTaskOption{PipelineNames: []string{name}, TaskCreators: filters, Type: typeString} + case "committer": + listTaskOpt = &commonrepo.ListTaskOption{PipelineName: name, Limit: maxResult, Skip: startAt, Committers: filters, ForWorkflowTaskList: true, Type: typeString} + countTaskOpt = &commonrepo.CountTaskOption{PipelineNames: []string{name}, Committers: filters, Type: typeString} + case "serviceName": + listTaskOpt = &commonrepo.ListTaskOption{PipelineName: name, Limit: maxResult, Skip: startAt, ServiceModule: filters, ForWorkflowTaskList: true, Type: typeString} + countTaskOpt = &commonrepo.CountTaskOption{PipelineNames: []string{name}, ServiceModule: filters, Type: typeString} + case "taskStatus": + listTaskOpt = &commonrepo.ListTaskOption{PipelineName: name, Limit: maxResult, Skip: startAt, Statuses: filters, ForWorkflowTaskList: true, Type: typeString} + countTaskOpt = &commonrepo.CountTaskOption{PipelineNames: []string{name}, Statuses: filters, Type: typeString} + default: + listTaskOpt = &commonrepo.ListTaskOption{PipelineName: name, Limit: maxResult, Skip: startAt, ForWorkflowTaskList: true, Type: typeString} + countTaskOpt = &commonrepo.CountTaskOption{PipelineNames: []string{name}, Type: typeString} + } + restp, err = commonrepo.NewTaskColl().List(listTaskOpt) if err != nil { - log.Errorf("PipelineTaskV2.List error: %v", err) + log.Errorf("PipelineTaskV2.List: %v error: %s", listTaskOpt, err) return ret, e.ErrListTasks } - // 获取任务的服务列表 - var buildStage, deployStage *commonmodels.Stage - for _, t := range ret.Data { - t.BuildServices = []string{} + for _, t := range restp { + if t.WorkflowArgs == nil { + continue + } + t.Namespace = t.WorkflowArgs.Namespace + serviceModuleMap := make(map[string]*commonrepo.ServiceModule) - for _, stage := range t.Stages { - if stage.TaskType == config.TaskBuild { - buildStage = stage - } - if stage.TaskType == config.TaskDeploy { - deployStage = stage + for _, target := range t.WorkflowArgs.Target { + sm := &commonrepo.ServiceModule{ + ServiceName: target.ServiceName, + ServiceModule: target.Name, } + serviceModuleMap[fmt.Sprintf("%s_%s", target.Name, target.ServiceName)] = sm + t.ServiceModules = append(t.ServiceModules, sm) } - // 有构建返回构建的服务，没构建返回部署的服务 - if buildStage != nil { - for serviceName := range buildStage.SubTasks { - t.BuildServices = append(t.BuildServices, serviceName) + for _, stage := range t.Stages { + if stage.TaskType != config.TaskBuild { + continue } - } else if deployStage != nil { - for serviceName := range deployStage.SubTasks { - t.BuildServices = append(t.BuildServices, serviceName) + for fullServiceName, sTask := range stage.SubTasks { + buildTask, err := base.ToBuildTask(sTask) + if err != nil { + log.Warnf("failed to get build task for task: %s, err: %s", name, err) + continue + } + if sm, ok := serviceModuleMap[fullServiceName]; ok { + for _, buildInfo := range buildTask.JobCtx.Builds { + sm.CodeInfo = append(sm.CodeInfo, buildInfo) + } + } } + break } - // 清理，下次循环再使用 - buildStage = nil - deployStage = nil + t.WorkflowArgs = nil + t.Stages = nil } - pipelineList := []string{name} - ret.Total, err = commonrepo.NewTaskColl().Count(&commonrepo.CountTaskOption{PipelineNames: pipelineList, Type: typeString}) + ret.Data = restp + ret.Total, err = commonrepo.NewTaskColl().Count(countTaskOpt) if err != nil { - log.Errorf("PipelineTaskV2.List error: %v", err) + log.Errorf("PipelineTaskV2.List Count error: %v", err) return ret, e.ErrCountTasks } return ret, nil @@ -350,6 +387,38 @@ func GetPipelineTaskV2(taskID int64, pipelineName string, typeString config.Pipe return resp, nil } +func GetFiltersPipelineTaskV2(projectName, pipelineName, querytype string, typeString config.PipelineType, log *zap.SugaredLogger) ([]interface{}, error) { + resp := []interface{}{} + var err error + fieldName := "" + switch querytype { + case "creator": + fieldName = "task_creator" + resp, err = commonrepo.NewTaskColl().DistinctFieldsPipelineTask(fieldName, projectName, pipelineName, typeString, false) + if err != nil { + log.Errorf("[%s] DistinctFeildsPipelineTask fieldName: %s error: %s", fieldName, pipelineName, err) + return resp, e.ErrGetTask + } + case "committer": + fieldName = "workflow_args.committer" + resp, err = commonrepo.NewTaskColl().DistinctFieldsPipelineTask(fieldName, projectName, pipelineName, typeString, false) + if err != nil { + log.Errorf("[%s] DistinctFeildsPipelineTask fieldName: %s error: %s", fieldName, pipelineName, err) + return resp, e.ErrGetTask + } + case "serviceName": + fieldName = "workflow_args.targets.name" + resp, err = commonrepo.NewTaskColl().DistinctFieldsPipelineTask(fieldName, projectName, pipelineName, typeString, false) + if err != nil { + log.Errorf("[%s] DistinctFeildsPipelineTask fieldName: %s error: %s", fieldName, pipelineName, err) + return resp, e.ErrGetTask + } + default: + return resp, fmt.Errorf("queryType parameter is invalid") + } + return resp, nil +} + func RestartPipelineTaskV2(userName string, taskID int64, pipelineName string, typeString config.PipelineType, log *zap.SugaredLogger) error { t, err := commonrepo.NewTaskColl().Find(taskID, pipelineName, typeString) if err != nil { @@ -385,7 +454,7 @@ func RestartPipelineTaskV2(userName string, taskID int64, pipelineName string, t subBuildTaskMap := subStage.SubTasks for serviceModule, subTask := range subBuildTaskMap { if buildInfo, err := base.ToBuildTask(subTask); err == nil { - if newModules, err := commonrepo.NewBuildColl().List(&commonrepo.BuildListOption{Targets: []string{serviceModule}, ServiceName: buildInfo.Service, ProductName: t.ProductName}); err == nil { + if newModules, err := commonrepo.NewBuildColl().List(&commonrepo.BuildListOption{Targets: []string{serviceModule}, ServiceName: buildInfo.Service, ProductName: t.ProductName}); err == nil && len(newModules) > 0 { newBuildInfo := newModules[0] buildInfo.JobCtx.BuildSteps = []*task.BuildStep{} if newBuildInfo.Scripts != "" { @@ -412,6 +481,7 @@ func RestartPipelineTaskV2(userName string, taskID int64, pipelineName string, t buildInfo.BuildOS = newBuildInfo.PreBuild.BuildOS buildInfo.ImageFrom = newBuildInfo.PreBuild.ImageFrom buildInfo.ResReq = newBuildInfo.PreBuild.ResReq + buildInfo.ResReqSpec = newBuildInfo.PreBuild.ResReqSpec } if newBuildInfo.PostBuild != nil && newBuildInfo.PostBuild.DockerBuild != nil { @@ -476,6 +546,7 @@ func RestartPipelineTaskV2(userName string, taskID int64, pipelineName string, t testInfo.BuildOS = newTestInfo.PreTest.BuildOS testInfo.ImageFrom = newTestInfo.PreTest.ImageFrom testInfo.ResReq = newTestInfo.PreTest.ResReq + testInfo.ResReqSpec = newTestInfo.PreTest.ResReqSpec } // 设置 build 安装脚本 testInfo.InstallCtx, err = buildInstallCtx(testInfo.InstallItems) @@ -492,8 +563,10 @@ func RestartPipelineTaskV2(userName string, taskID int64, pipelineName string, t } case config.TaskDeploy: resetImage := false + resetImagePolicy := setting.ResetImagePolicyTaskCompletedOrder if workflow, err := commonrepo.NewWorkflowColl().Find(t.PipelineName); err == nil { resetImage = workflow.ResetImage + resetImagePolicy = workflow.ResetImagePolicy } timeout := 0 if productTempl, err := template.NewProductColl().Find(t.ProductName); err == nil { @@ -506,6 +579,7 @@ func RestartPipelineTaskV2(userName string, taskID int64, pipelineName string, t deployInfo.Timeout = timeout deployInfo.IsRestart = true deployInfo.ResetImage = resetImage + deployInfo.ResetImagePolicy = resetImagePolicy if newDeployInfo, err := deployInfo.ToSubTask(); err == nil { subDeployTaskMap[serviceName] = newDeployInfo } @@ -527,6 +601,12 @@ func RestartPipelineTaskV2(userName string, taskID int64, pipelineName string, t func TestArgsToTestSubtask(args *commonmodels.TestTaskArgs, pt *task.Task, log *zap.SugaredLogger) (*task.Testing, error) { var resp *task.Testing + testTask := &task.Testing{ + TaskType: config.TaskTestingV2, + Enabled: true, + TestName: "test", + } + allTestings, err := commonrepo.NewTestingColl().List(&commonrepo.ListTestOption{ProductName: args.ProductName, TestType: ""}) if err != nil { log.Errorf("testArgsToTestSubtask TestingModule.List error: %v", err) @@ -541,7 +621,13 @@ func TestArgsToTestSubtask(args *commonmodels.TestTaskArgs, pt *task.Task, log * } else { testArg.Builds = testing.Repos pr, _ := strconv.Atoi(args.MergeRequestID) - testArg.Builds[0].PR = pr + + for i, build := range testArg.Builds { + if build.Source == args.Source { + testArg.Builds[i].PR = pr + } + } + } if testing.PreTest != nil { @@ -549,6 +635,29 @@ func TestArgsToTestSubtask(args *commonmodels.TestTaskArgs, pt *task.Task, log * } testArg.TestModuleName = args.TestName + + // In some old testing configurations, the `pre_test.cluster_id` field is empty indicating that's a local cluster. + // We do a protection here to avoid query failure. + // Resaving the testing configuration after v1.8.0 will automatically populate this field. + if testing.PreTest.ClusterID == "" { + testing.PreTest.ClusterID = setting.LocalClusterID + } + + clusterInfo, err := commonrepo.NewK8SClusterColl().Get(testing.PreTest.ClusterID) + if err != nil { + return resp, e.ErrListTestModule.AddDesc(err.Error()) + } + testTask.Cache = clusterInfo.Cache + + // If the cluster is not configured with a cache medium, the cache cannot be used, so don't enable cache explicitly. + if testTask.Cache.MediumType == "" { + testTask.CacheEnable = false + } else { + testTask.CacheEnable = testing.CacheEnable + testTask.CacheDirType = testing.CacheDirType + testTask.CacheUserDir = testing.CacheUserDir + } + break } } @@ -558,23 +667,20 @@ func TestArgsToTestSubtask(args *commonmodels.TestTaskArgs, pt *task.Task, log * log.Errorf("[%s]get TestingModule error: %v", args.TestName, err) return resp, err } - testTask := &task.Testing{ - TaskType: config.TaskTestingV2, - Enabled: true, - TestName: "test", - Timeout: testModule.Timeout, - } + testTask.Timeout = testModule.Timeout + testTask.TestModuleName = testModule.Name testTask.JobCtx.TestType = testModule.TestType testTask.JobCtx.Builds = testModule.Repos testTask.JobCtx.BuildSteps = append(testTask.JobCtx.BuildSteps, &task.BuildStep{BuildType: "shell", Scripts: testModule.Scripts}) testTask.JobCtx.TestResultPath = testModule.TestResultPath + testTask.JobCtx.TestReportPath = testModule.TestReportPath testTask.JobCtx.TestThreshold = testModule.Threshold testTask.JobCtx.Caches = testModule.Caches testTask.JobCtx.ArtifactPaths = testModule.ArtifactPaths if testTask.Registries == nil { - registries, err := commonservice.ListRegistryNamespaces(log) + registries, err := commonservice.ListRegistryNamespaces("", true, log) if err != nil { log.Errorf("ListRegistryNamespaces err:%v", err) } else { @@ -585,6 +691,8 @@ func TestArgsToTestSubtask(args *commonmodels.TestTaskArgs, pt *task.Task, log * testTask.InstallItems = testModule.PreTest.Installs testTask.JobCtx.CleanWorkspace = testModule.PreTest.CleanWorkspace testTask.JobCtx.EnableProxy = testModule.PreTest.EnableProxy + testTask.Namespace = testModule.PreTest.Namespace + testTask.ClusterID = testModule.PreTest.ClusterID envs := testModule.PreTest.Envs[:] @@ -598,11 +706,13 @@ func TestArgsToTestSubtask(args *commonmodels.TestTaskArgs, pt *task.Task, log * } } envs = append(envs, &commonmodels.KeyVal{Key: "TEST_URL", Value: GetLink(pt, configbase.SystemAddress(), config.TestType)}) + envs = append(envs, &commonmodels.KeyVal{Key: "WORKSPACE", Value: "/workspace"}) testTask.JobCtx.EnvVars = envs testTask.ImageID = testModule.PreTest.ImageID testTask.BuildOS = testModule.PreTest.BuildOS testTask.ImageFrom = testModule.PreTest.ImageFrom testTask.ResReq = testModule.PreTest.ResReq + testTask.ResReqSpec = testModule.PreTest.ResReqSpec } // 设置 build 安装脚本 testTask.InstallCtx, err = buildInstallCtx(testTask.InstallItems) @@ -694,6 +804,9 @@ func EnsureTestingResp(mt *commonmodels.Testing) { if len(mt.Repos) == 0 { mt.Repos = make([]*types.Repository, 0) } + for _, repo := range mt.Repos { + repo.RepoNamespace = repo.GetRepoNamespace() + } if mt.PreTest != nil { if len(mt.PreTest.Installs) == 0 { @@ -923,20 +1036,18 @@ func GePackageFileContent(pipelineName string, taskID int64, log *zap.SugaredLog return fileBytes, packageFile, err } -func GetArtifactFileContent(pipelineName string, taskID int64, log *zap.SugaredLogger) ([]byte, error) { - s3Storage, artifactFiles, _ := GetTestArtifactInfo(pipelineName, "", taskID, log) - tempdir, _ := ioutil.TempDir("", "") - sourcePath := path.Join(tempdir, "artifact") - if _, err := os.Stat(sourcePath); os.IsNotExist(err) { - _ = os.MkdirAll(sourcePath, 0777) +func GetArtifactFileContent(pipelineName string, taskID int64, notHistoryFileFlag bool, log *zap.SugaredLogger) ([]byte, error) { + s3Storage, client, artifactFiles, artifactResultOutByts, err := GetArtifactAndS3Info(pipelineName, "", taskID, notHistoryFileFlag, log) + if err != nil { + return nil, fmt.Errorf("download artifact err: %s", err) } - forcedPathStyle := true - if s3Storage.Provider == setting.ProviderSourceAli { - forcedPathStyle = false + if notHistoryFileFlag { + return artifactResultOutByts, nil } - client, err := s3tool.NewClient(s3Storage.Endpoint, s3Storage.Ak, s3Storage.Sk, s3Storage.Insecure, forcedPathStyle) - if err != nil { - return nil, fmt.Errorf("failed to create s3 client , error: %v", err) + tempDir, _ := ioutil.TempDir("", "") + sourcePath := path.Join(tempDir, "artifact") + if _, err := os.Stat(sourcePath); os.IsNotExist(err) { + _ = os.MkdirAll(sourcePath, 0777) } for _, artifactFile := range artifactFiles { @@ -948,12 +1059,10 @@ func GetArtifactFileContent(pipelineName string, taskID int64, log *zap.SugaredL return nil, fmt.Errorf("failed to create file %s %v", artifactFileName, err) } defer func() { - file.Close() - os.Remove(path.Join(sourcePath, artifactFileName)) + _ = file.Close() }() - objectKey := s3Storage.GetObjectPath(artifactFile) - err = client.Download(s3Storage.Bucket, objectKey, file.Name()) + err = client.Download(s3Storage.Bucket, artifactFile, file.Name()) if err != nil { return nil, fmt.Errorf("failed to download %s %v", artifactFile, err) } @@ -968,20 +1077,20 @@ func GetArtifactFileContent(pipelineName string, taskID int64, log *zap.SugaredL } defer func() { _ = os.Remove(artifactTarFileName) - _ = os.Remove(tempdir) + _ = os.Remove(tempDir) }() fileBytes, err := ioutil.ReadFile(path.Join(sourcePath, "artifact.tar.gz")) return fileBytes, err } -func GetTestArtifactInfo(pipelineName, dir string, taskID int64, log *zap.SugaredLogger) (*s3.S3, []string, error) { +func GetArtifactAndS3Info(pipelineName, dir string, taskID int64, notHistoryFileFlag bool, log *zap.SugaredLogger) (*s3.S3, *s3tool.Client, []string, []byte, error) { fis := make([]string, 0) storage, err := s3.FindDefaultS3() if err != nil { log.Errorf("GetTestArtifactInfo FindDefaultS3 err:%v", err) - return nil, fis, nil + return nil, nil, fis, nil, err } if storage.Subfolder != "" { @@ -996,13 +1105,29 @@ func GetTestArtifactInfo(pipelineName, dir string, taskID int64, log *zap.Sugare client, err := s3tool.NewClient(storage.Endpoint, storage.Ak, storage.Sk, storage.Insecure, forcedPathStyle) if err != nil { log.Errorf("GetTestArtifactInfo Create S3 client err:%+v", err) - return nil, fis, nil + return nil, nil, fis, nil, err } + + if notHistoryFileFlag { + objectKey := storage.GetObjectPath(fmt.Sprintf("%s/%s/%s", dir, "workspace", setting.ArtifactResultOut)) + object, err := client.GetFile(storage.Bucket, objectKey, &s3tool.DownloadOption{RetryNum: 2}) + if err != nil { + log.Errorf("GetTestArtifactInfo GetFile err:%s", err) + return nil, nil, fis, nil, err + } + fileByts, err := ioutil.ReadAll(object.Body) + if err != nil { + log.Errorf("GetTestArtifactInfo ioutil.ReadAll err:%s", err) + return nil, nil, fis, nil, err + } + return storage, client, fis, fileByts, nil + } + prefix := storage.GetObjectPath(dir) files, err := client.ListFiles(storage.Bucket, prefix, true) if err != nil || len(files) <= 0 { log.Errorf("GetTestArtifactInfo ListFiles err:%v", err) - return nil, fis, nil + return nil, nil, fis, nil, err } - return storage, files, nil + return storage, client, files, nil, nil } diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_validation.go b/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_validation.go index f233e98f1cfcdec6520ad283f58518a63cf23599..cff9b4f287842968fbf54359adbdbb4fe1bc28ac 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_validation.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_validation.go @@ -22,32 +22,27 @@ import ( "fmt" "regexp" "sort" - "strconv" - "strings" "sync" - "time" "github.com/google/go-github/v35/github" "go.uber.org/zap" + appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/labels" - "sigs.k8s.io/controller-runtime/pkg/client" configbase "github.com/koderover/zadig/pkg/config" "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" - templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/codehub" git "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/github" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/codehost" - "github.com/koderover/zadig/pkg/shared/kube/wrapper" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/gitee" "github.com/koderover/zadig/pkg/tool/kube/getter" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/types" @@ -57,6 +52,7 @@ import ( const ( NameSpaceRegexString = "[^a-z0-9.-]" defaultNameRegexString = "^[a-zA-Z0-9-_]{1,50}$" + InterceptCommitID = 8 ) var ( @@ -257,10 +253,7 @@ func FmtBuilds(builds []*types.Repository, log *zap.SugaredLogger) { log.Error("codehostID can't be empty") return } - opt := &codehost.Option{ - CodeHostID: cID, - } - detail, err := codehost.GetCodeHostInfo(opt) + detail, err := systemconfig.New().GetCodeHost(cID) if err != nil { log.Error(err) return @@ -297,33 +290,30 @@ func SetTriggerBuilds(builds []*types.Repository, buildArgs []*types.Repository, go func(build *types.Repository) { defer wg.Done() - setBuildInfo(build, log) + setBuildInfo(build, buildArgs, log) }(build) } wg.Wait() return nil } -func setBuildInfo(build *types.Repository, log *zap.SugaredLogger) { - opt := &codehost.Option{ - CodeHostID: build.CodehostID, - } - codeHostInfo, err := codehost.GetCodeHostInfo(opt) +func setBuildInfo(build *types.Repository, buildArgs []*types.Repository, log *zap.SugaredLogger) { + codeHostInfo, err := systemconfig.New().GetCodeHost(build.CodehostID) if err != nil { log.Errorf("failed to get codehost detail %d %v", build.CodehostID, err) return } - if codeHostInfo.Type == codehost.GitLabProvider || codeHostInfo.Type == codehost.GerritProvider || codeHostInfo.Type == codehost.IlyshinProvider { + if codeHostInfo.Type == systemconfig.GitLabProvider || codeHostInfo.Type == systemconfig.GerritProvider { if build.CommitID == "" { var commit *RepoCommit var pr *PRCommit var err error if build.Tag != "" { - commit, err = QueryByTag(build.CodehostID, build.RepoOwner, build.RepoName, build.Tag, log) + commit, err = QueryByTag(build.CodehostID, build.GetRepoNamespace(), build.RepoName, build.Tag, log) } else if build.Branch != "" && build.PR == 0 { - commit, err = QueryByBranch(build.CodehostID, build.RepoOwner, build.RepoName, build.Branch, log) + commit, err = QueryByBranch(build.CodehostID, build.GetRepoNamespace(), build.RepoName, build.Branch, log) } else if build.PR > 0 { - pr, err = GetLatestPrCommit(build.CodehostID, build.PR, build.RepoOwner, build.RepoName, log) + pr, err = GetLatestPrCommit(build.CodehostID, build.PR, build.GetRepoNamespace(), build.RepoName, log) if err == nil && pr != nil { commit = &RepoCommit{ ID: pr.ID, @@ -348,8 +338,8 @@ func setBuildInfo(build *types.Repository, log *zap.SugaredLogger) { build.CommitMessage = commit.Message build.AuthorName = commit.AuthorName } - } else if codeHostInfo.Type == codehost.CodeHubProvider { - codeHubClient := codehub.NewClient(codeHostInfo.AccessKey, codeHostInfo.SecretKey, codeHostInfo.Region) + } else if codeHostInfo.Type == systemconfig.CodeHubProvider { + codeHubClient := codehub.NewClient(codeHostInfo.AccessKey, codeHostInfo.SecretKey, codeHostInfo.Region, config.ProxyHTTPSAddr(), codeHostInfo.EnableProxy) if build.CommitID == "" && build.Branch != "" { branchList, _ := codeHubClient.BranchList(build.RepoUUID) for _, branchInfo := range branchList { @@ -361,31 +351,85 @@ func setBuildInfo(build *types.Repository, log *zap.SugaredLogger) { } } } - } else { - gitCli := git.NewClient(codeHostInfo.AccessToken, config.ProxyHTTPSAddr()) - //// 需要后端自动获取Branch当前Commit，并填写到build中 + } else if codeHostInfo.Type == systemconfig.GiteeProvider { + gitCli := gitee.NewClient(codeHostInfo.ID, codeHostInfo.AccessToken, config.ProxyHTTPSAddr(), codeHostInfo.EnableProxy) if build.CommitID == "" { - // 如果是仅填写Branch编译 - if build.Branch != "" && build.PR == 0 { - branch, _, err := gitCli.Repositories.GetBranch(context.Background(), build.RepoOwner, build.RepoName, build.Branch) - if err == nil { - build.CommitID = *branch.Commit.SHA - build.CommitMessage = *branch.Commit.Commit.Message - build.AuthorName = *branch.Commit.Commit.Author.Name + if build.Tag != "" && build.PR == 0 { + tags, err := gitCli.ListTags(context.Background(), codeHostInfo.AccessToken, build.RepoOwner, build.RepoName) + if err != nil { + log.Errorf("failed to gitee ListTags err:%s", err) + return + } + + for _, tag := range tags { + if tag.Name == build.Tag { + build.CommitID = tag.Commit.Sha + commitInfo, err := gitCli.GetSingleCommitOfProject(context.Background(), codeHostInfo.AccessToken, build.RepoOwner, build.RepoName, build.CommitID) + if err != nil { + log.Errorf("failed to gitee GetCommit %s err:%s", tag.Commit.Sha, err) + return + } + build.CommitMessage = commitInfo.Commit.Message + build.AuthorName = commitInfo.Commit.Author.Name + return + } + } + } else if build.Branch != "" && build.PR == 0 { + branch, err := gitCli.GetSingleBranch(codeHostInfo.AccessToken, build.RepoOwner, build.RepoName, build.Branch) + if err != nil { + log.Errorf("failed to gitee GetSingleBranch repoOwner:%s,repoName:%s,repoBranch:%s err:%s", build.RepoOwner, build.RepoName, build.Branch, err) + return } - } else if build.Tag != "" && build.PR == 0 { + build.CommitID = branch.Commit.Sha + build.CommitMessage = branch.Commit.Commit.Message + build.AuthorName = branch.Commit.Commit.Author.Name + } else if build.PR > 0 { + prCommits, err := gitCli.ListCommits(context.Background(), build.RepoOwner, build.RepoName, build.PR, nil) + sort.SliceStable(prCommits, func(i, j int) bool { + return prCommits[i].Commit.Committer.Date.Unix() > prCommits[j].Commit.Committer.Date.Unix() + }) + if err == nil && len(prCommits) > 0 { + for _, commit := range prCommits { + build.CommitID = commit.Sha + build.CommitMessage = commit.Commit.Message + build.AuthorName = commit.Commit.Author.Name + return + } + } + } + } + } else if codeHostInfo.Type == systemconfig.GitHubProvider { + gitCli := git.NewClient(codeHostInfo.AccessToken, config.ProxyHTTPSAddr(), codeHostInfo.EnableProxy) + if build.CommitID == "" { + if build.Tag != "" && build.PR == 0 { opt := &github.ListOptions{Page: 1, PerPage: 100} tags, _, err := gitCli.Repositories.ListTags(context.Background(), build.RepoOwner, build.RepoName, opt) - if err == nil { - for _, tag := range tags { - if *tag.Name == build.Tag { - build.CommitID = *tag.Commit.SHA - build.CommitMessage = *tag.Commit.Message - build.AuthorName = *tag.Commit.Author.Name - return + if err != nil { + log.Errorf("failed to github ListTags err:%s", err) + return + } + for _, tag := range tags { + if *tag.Name == build.Tag { + build.CommitID = tag.Commit.GetSHA() + if tag.Commit.Message == nil { + commitInfo, _, err := gitCli.Repositories.GetCommit(context.Background(), build.RepoOwner, build.RepoName, build.CommitID) + if err != nil { + log.Errorf("failed to github GetCommit %s err:%s", tag.Commit.GetURL(), err) + return + } + build.CommitMessage = commitInfo.GetCommit().GetMessage() } + build.AuthorName = tag.Commit.GetAuthor().GetName() + return } } + } else if build.Branch != "" && build.PR == 0 { + branch, _, err := gitCli.Repositories.GetBranch(context.Background(), build.RepoOwner, build.RepoName, build.Branch) + if err == nil { + build.CommitID = *branch.Commit.SHA + build.CommitMessage = *branch.Commit.Commit.Message + build.AuthorName = *branch.Commit.Commit.Author.Name + } } else if build.PR > 0 { opt := &github.ListOptions{Page: 1, PerPage: 100} prCommits, _, err := gitCli.PullRequests.ListCommits(context.Background(), build.RepoOwner, build.RepoName, build.PR, opt) @@ -404,28 +448,18 @@ func setBuildInfo(build *types.Repository, log *zap.SugaredLogger) { log.Warnf("github setBuildInfo failed, use build %+v", build) return } - //if build.Branch != "" && build.PR > 0 { - // prCommits := make([]*github.RepositoryCommit, 0) - // opt := &github.ListOptions{Page: 1, PerPage: 100} - // for opt.Page > 0 { - // list, resp, err := gitCli.PullRequests.ListCommits(context.Background(), build.RepoOwner, build.RepoName, build.PR, opt) - // if err != nil { - // fmt.Printf("ListCommits error: %v\n", err) - // return - // } - // prCommits = append(prCommits, list...) - // opt.Page = resp.NextPage - // } - // - // if len(prCommits) > 0 { - // commit := prCommits[len(prCommits)-1] - // build.CommitID = *commit.SHA - // } - //} } - + } else if codeHostInfo.Type == systemconfig.OtherProvider { + build.SSHKey = codeHostInfo.SSHKey + build.PrivateAccessToken = codeHostInfo.PrivateAccessToken + for _, buildArg := range buildArgs { + if buildArg.RepoOwner == build.RepoOwner && buildArg.RepoName == build.RepoName { + setBuildFromArg(build, buildArg) + break + } + } + return } - } // 根据传入的build arg设置build参数 @@ -479,117 +513,19 @@ func setManunalBuilds(builds []*types.Repository, buildArgs []*types.Repository, break } } - setBuildInfo(build, log) + setBuildInfo(build, buildArgs, log) }(build) } wg.Wait() return nil } -// releaseCandidate 根据 TaskID 生成编译镜像Tag或者二进制包后缀 -// TODO: max length of a tag is 128 -func releaseCandidate(b *task.Build, taskID int64, productName, envName, deliveryType string) string { - timeStamp := time.Now().Format("20060102150405") - if len(b.JobCtx.Builds) == 0 { - switch deliveryType { - case config.TarResourceType: - return fmt.Sprintf("%s-%s", b.ServiceName, timeStamp) - default: - return fmt.Sprintf("%s:%s", b.ServiceName, timeStamp) - } - } - - first := b.JobCtx.Builds[0] - for index, build := range b.JobCtx.Builds { - if build.IsPrimary { - first = b.JobCtx.Builds[index] - } - } - - // 替换 Tag 和 Branch 中的非法字符为 "-", 避免 docker build 失败 - var ( - reg = regexp.MustCompile(`[^\w.-]`) - customImageRule *template.CustomRule - customTarRule *template.CustomRule - ) - - if project, err := templaterepo.NewProductColl().Find(productName); err != nil { - log.Errorf("find project err:%s", err) - } else { - customImageRule = project.CustomImageRule - customTarRule = project.CustomTarRule - } - - candidate := &candidate{ - Branch: string(reg.ReplaceAll([]byte(first.Branch), []byte("-"))), - CommitID: first.CommitID, - PR: first.PR, - Tag: string(reg.ReplaceAll([]byte(first.Tag), []byte("-"))), - EnvName: envName, - Timestamp: timeStamp, - TaskID: taskID, - ProductName: productName, - ServiceName: b.ServiceName, - } - switch deliveryType { - case config.TarResourceType: - return replaceVariable(customTarRule, candidate) - default: - return replaceVariable(customImageRule, candidate) - } -} - -type candidate struct { - Branch string - Tag string - CommitID string - PR int - TaskID int64 - Timestamp string - ProductName string - ServiceName string - EnvName string -} - -// There are four situations in total -// 1.Execute workflow selection tag build -// 2.Execute workflow selection branch and pr build -// 3.Execute workflow selection branch pr build -// 4.Execute workflow selection branch build -func replaceVariable(customRule *template.CustomRule, candidate *candidate) string { - var currentRule string - if candidate.Tag != "" { - if customRule == nil { - return fmt.Sprintf("%s:%s-%s", candidate.ServiceName, candidate.Timestamp, candidate.Tag) - } - currentRule = customRule.TagRule - } else if candidate.Branch != "" && candidate.PR != 0 { - if customRule == nil { - return fmt.Sprintf("%s:%s-%d-%s-pr-%d", candidate.ServiceName, candidate.Timestamp, candidate.TaskID, candidate.Branch, candidate.PR) - } - currentRule = customRule.PRAndBranchRule - } else if candidate.Branch == "" && candidate.PR != 0 { - if customRule == nil { - return fmt.Sprintf("%s:%s-%d-pr-%d", candidate.ServiceName, candidate.Timestamp, candidate.TaskID, candidate.PR) - } - currentRule = customRule.PRRule - } else if candidate.Branch != "" && candidate.PR == 0 { - if customRule == nil { - return fmt.Sprintf("%s:%s-%d-%s", candidate.ServiceName, candidate.Timestamp, candidate.TaskID, candidate.Branch) - } - currentRule = customRule.BranchRule - } - - currentRule = commonservice.ReplaceRuleVariable(currentRule, &commonservice.Variable{ - candidate.ServiceName, candidate.Timestamp, strconv.FormatInt(candidate.TaskID, 10), candidate.CommitID, candidate.ProductName, candidate.EnvName, - candidate.Tag, candidate.Branch, strconv.Itoa(candidate.PR), - }) - return currentRule -} - // GetImage suffix 可以是 branch name 或者 pr number func GetImage(registry *commonmodels.RegistryNamespace, suffix string) string { - return fmt.Sprintf("%s/%s/%s", util.GetURLHostName(registry.RegAddr), registry.Namespace, suffix) + if registry.RegProvider == config.RegistryTypeAWS { + return fmt.Sprintf("%s/%s", util.TrimURLScheme(registry.RegAddr), suffix) + } + return fmt.Sprintf("%s/%s/%s", util.TrimURLScheme(registry.RegAddr), registry.Namespace, suffix) } // GetPackageFile suffix 可以是 branch name 或者 pr number @@ -609,6 +545,7 @@ func prepareTaskEnvs(pt *task.Task, log *zap.SugaredLogger) []*commonmodels.KeyV &commonmodels.KeyVal{Key: "IMAGE", Value: pt.TaskArgs.Deploy.Image}, &commonmodels.KeyVal{Key: "PKG_FILE", Value: pt.TaskArgs.Deploy.PackageFile}, &commonmodels.KeyVal{Key: "LOG_FILE", Value: "/tmp/user_script.log"}, + &commonmodels.KeyVal{Key: "WORKSPACE", Value: "/workspace"}, ) // 设置编译模块参数化配置信息 @@ -667,9 +604,9 @@ func SetCandidateRegistry(payload *commonmodels.ConfigPayload, log *zap.SugaredL return nil } - reg, err := commonservice.FindDefaultRegistry(log) + reg, _, err := commonservice.FindDefaultRegistry(true, log) if err != nil { - log.Errorf("can't find default candidate registry: %v", err) + log.Errorf("can't find default candidate registry: %s", err) return e.ErrFindRegistry.AddDesc(err.Error()) } @@ -680,8 +617,8 @@ func SetCandidateRegistry(payload *commonmodels.ConfigPayload, log *zap.SugaredL return nil } -// validateServiceContainer validate container with envName like dev -func validateServiceContainer(envName, productName, serviceName, container string) (string, error) { +// getImageInfoFromWorkload find the current image info from the cluster +func getImageInfoFromWorkload(envName, productName, serviceName, container string) (string, error) { product, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ Name: productName, EnvName: envName, @@ -690,14 +627,79 @@ func validateServiceContainer(envName, productName, serviceName, container strin return "", err } - kubeClient, err := kube.GetKubeClient(product.ClusterID) + serviceInfo, err := commonrepo.NewServiceColl().Find(&commonrepo.ServiceFindOption{ + ServiceName: serviceName, + ProductName: productName, + }) if err != nil { return "", err } - return validateServiceContainer2( - product.Namespace, envName, productName, serviceName, container, product.Source, kubeClient, - ) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), product.ClusterID) + if err != nil { + return "", err + } + + if product.Source == setting.SourceFromHelm { + return findCurrentlyUsingImage(product, serviceName, container) + } + + switch serviceInfo.WorkloadType { + case setting.StatefulSet: + var statefulSet *appsv1.StatefulSet + statefulSet, _, err = getter.GetStatefulSet(product.Namespace, serviceName, kubeClient) + if err != nil { + return "", err + } + for _, c := range statefulSet.Spec.Template.Spec.Containers { + if c.Name == container { + return c.Image, nil + } + } + return "", errors.New("no container in statefulset found") + case setting.Deployment: + var deployment *appsv1.Deployment + deployment, _, err = getter.GetDeployment(product.Namespace, serviceName, kubeClient) + if err != nil { + return "", err + } + for _, c := range deployment.Spec.Template.Spec.Containers { + if c.Name == container { + return c.Image, nil + } + } + return "", errors.New("no container in deployment found") + default: + // since in some version of the service, there are no workload type, we need to do something with this + selector := labels.Set{setting.ProductLabel: product.ProductName, setting.ServiceLabel: serviceName}.AsSelector() + var deployments []*appsv1.Deployment + deployments, err = getter.ListDeployments(product.Namespace, selector, kubeClient) + if err != nil { + return "", err + } + for _, deploy := range deployments { + for _, c := range deploy.Spec.Template.Spec.Containers { + if c.Name == container { + return c.Image, nil + } + } + } + var statefulSets []*appsv1.StatefulSet + statefulSets, err = getter.ListStatefulSets(product.Namespace, selector, kubeClient) + if err != nil { + return "", err + } + for _, sts := range statefulSets { + for _, c := range sts.Spec.Template.Spec.Containers { + if c.Name == container { + return c.Image, nil + } + } + } + + log.Errorf("No workload found with container: %s", container) + return "", errors.New("no workload found with container") + } } type ContainerNotFound struct { @@ -718,38 +720,18 @@ func (c *ImageIllegal) Error() string { return "" } -// validateServiceContainer2 validate container with raw namespace like dev-product -func validateServiceContainer2(namespace, envName, productName, serviceName, container, source string, kubeClient client.Client) (string, error) { - var selector labels.Selector - - //helm和托管类型的服务查询所有标签的pod - if source != setting.SourceFromHelm && source != setting.SourceFromExternal { - selector = labels.Set{setting.ProductLabel: productName, setting.ServiceLabel: serviceName}.AsSelector() - //builder := &SelectorBuilder{ProductName: productName, ServiceName: serviceName} - //selector = builder.BuildSelector() - } - - pods, err := getter.ListPods(namespace, selector, kubeClient) - if err != nil { - return "", fmt.Errorf("[%s] ListPods %s/%s error: %v", namespace, productName, serviceName, err) - } - - for _, p := range pods { - pod := wrapper.Pod(p).Resource() - for _, c := range pod.ContainerStatuses { - if c.Name == container || strings.Contains(c.Image, container) { - return c.Image, nil +// find currently using image for services deployed by helm +func findCurrentlyUsingImage(productInfo *commonmodels.Product, serviceName, containerName string) (string, error) { + for _, service := range productInfo.GetServiceMap() { + if service.ServiceName == serviceName { + for _, container := range service.Containers { + if container.Name == containerName { + return container.Image, nil + } } } } - log.Errorf("[%s]container %s not found", namespace, container) - - return "", &ContainerNotFound{ - ServiceName: serviceName, - Container: container, - EnvName: envName, - ProductName: productName, - } + return "", fmt.Errorf("failed to find image url") } // IsProductAuthed 查询指定产品是否授权给用户, 或者用户所在的组 diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/plugin_repo.go b/pkg/microservice/aslan/core/workflow/service/workflow/plugin_repo.go new file mode 100644 index 0000000000000000000000000000000000000000..dd5adc848f06b5dd6540d97f75a57160f385aefd --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/service/workflow/plugin_repo.go @@ -0,0 +1,160 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package workflow + +import ( + "embed" + "fmt" + "io/fs" + "os" + "path" + + "github.com/google/uuid" + "go.uber.org/zap" + "gopkg.in/yaml.v3" + + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +const ( + OfficalRepoOwner = "koderover" + OfficalRepoName = "zadig" + OfficalRepoURL = "https://github.com/" + OfficalRepoOwner + "/" + OfficalRepoName + OfficalBranch = "main" +) + +//go:embed plugins +var officalPluginRepoFiles embed.FS + +func UpsertUserPluginRepository(args *commonmodels.PluginRepo, log *zap.SugaredLogger) error { + args.IsOffical = false + args.PluginTemplates = []*commonmodels.PluginTemplate{} + checkoutPath := fmt.Sprintf("/tmp/%s", uuid.New().String()) + defer os.RemoveAll(checkoutPath) + // TODO: git clone and checkout the plugin + plugins, err := loadPluginRepoInfos(checkoutPath, args.IsOffical, os.ReadDir, os.ReadFile) + if err != nil { + errMsg := fmt.Sprintf("load plugin from user user repo error: %s", err) + log.Error(errMsg) + return e.ErrUpsertPluginRepo.AddDesc(errMsg) + } + args.PluginTemplates = plugins + return commonrepo.NewPluginRepoColl().Upsert(args) +} + +func UpdateOfficalPluginRepository(log *zap.SugaredLogger) { + officalPluginRepo := &commonmodels.PluginRepo{ + RepoOwner: OfficalRepoOwner, + RepoName: OfficalRepoName, + Branch: OfficalBranch, + RepoURL: OfficalRepoURL, + IsOffical: true, + } + plugins, err := loadPluginRepoInfos("plugins", officalPluginRepo.IsOffical, officalPluginRepoFiles.ReadDir, officalPluginRepoFiles.ReadFile) + if err != nil { + log.Errorf("load offical plugin repo error: %v", err) + return + } + officalPluginRepo.PluginTemplates = plugins + if err := commonrepo.NewPluginRepoColl().Upsert(officalPluginRepo); err != nil { + log.Errorf("update offical plugin repo error: %v", err) + return + } +} + +type readDir func(name string) ([]fs.DirEntry, error) +type readFile func(name string) ([]byte, error) + +func loadPluginRepoInfos(baseDir string, isOffical bool, readDir readDir, readFile readFile) ([]*commonmodels.PluginTemplate, error) { + resp := []*commonmodels.PluginTemplate{} + dirs, err := readDir(baseDir) + if err != nil { + return resp, fmt.Errorf("loop plugin repo error: %v", err) + } + for _, dir := range dirs { + if !dir.IsDir() { + continue + } + subDirs, err := readDir(path.Join(baseDir, dir.Name())) + if err != nil { + return resp, fmt.Errorf("loop plugin repo dirs error: %v", err) + } + for _, subDir := range subDirs { + if !subDir.IsDir() { + continue + } + files, err := readDir(path.Join(baseDir, dir.Name(), subDir.Name())) + if err != nil { + return resp, fmt.Errorf("loop plugin repo sub dirs error: %v", err) + } + for _, file := range files { + if file.IsDir() { + continue + } + if file.Name() != dir.Name()+".yaml" { + continue + } + yamlFilebyte, err := readFile(path.Join(baseDir, dir.Name(), subDir.Name(), file.Name())) + if err != nil { + return resp, fmt.Errorf("read yaml files error: %v", err) + } + pluginTemplate := &commonmodels.PluginTemplate{} + if err := yaml.Unmarshal(yamlFilebyte, pluginTemplate); err != nil { + return resp, fmt.Errorf("unmarshal yaml files error: %v", err) + } + pluginTemplate.IsOffical = isOffical + resp = append(resp, pluginTemplate) + } + } + } + return resp, nil +} + +func ListPluginRepositories(log *zap.SugaredLogger) ([]*commonmodels.PluginRepo, error) { + resp, err := commonrepo.NewPluginRepoColl().List() + if err != nil { + log.Errorf("list Plugin repos error: %v", err) + return resp, e.ErrListPluginRepo.AddDesc(err.Error()) + } + return resp, nil +} + +func DeletePluginRepo(id string, log *zap.SugaredLogger) error { + if err := commonrepo.NewPluginRepoColl().Delete(id); err != nil { + log.Errorf("list Plugin repos error: %v", err) + return e.ErrListPluginRepo.AddDesc(err.Error()) + } + return nil +} + +func ListPluginTemplates(log *zap.SugaredLogger) ([]*commonmodels.PluginTemplate, error) { + resp := []*commonmodels.PluginTemplate{} + repos, err := commonrepo.NewPluginRepoColl().List() + if err != nil { + log.Errorf("list plugin templates error: %v", err) + return resp, e.ErrListPluginRepo.AddDesc(err.Error()) + } + for _, repo := range repos { + for _, template := range repo.PluginTemplates { + template.RepoURL = fmt.Sprintf("%s/%s", repo.RepoOwner, repo.RepoName) + resp = append(resp, template) + } + } + return resp, nil +} diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/plugins/README.md b/pkg/microservice/aslan/core/workflow/service/workflow/plugins/README.md new file mode 100644 index 0000000000000000000000000000000000000000..61e4f4e6a273533bc3fbf748fb159c5c18dfccfb --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/service/workflow/plugins/README.md @@ -0,0 +1,3 @@ +# zadig-plugin + +Zadig offical plugins diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/service_task.go b/pkg/microservice/aslan/core/workflow/service/workflow/service_task.go index a6aa7d81302b2f2d09454ffe834c6c628459d9ef..87384df1e5a59462c657403ec7b061d195535f63 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/service_task.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/service_task.go @@ -21,19 +21,18 @@ import ( "fmt" "sort" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" - "github.com/koderover/zadig/pkg/types" - "go.uber.org/zap" "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" + taskmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3" "github.com/koderover/zadig/pkg/setting" e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/types" ) const SplitSymbol = "&" @@ -90,7 +89,7 @@ func ListServiceWorkflows(productName, envName, serviceName, serviceType string, } // 获取支持升级此服务的产品工作流 - workflowOpt := &commonrepo.ListWorkflowOption{ProductName: productName} + workflowOpt := &commonrepo.ListWorkflowOption{Projects: []string{productName}} workflows, err := commonrepo.NewWorkflowColl().List(workflowOpt) if err != nil { log.Errorf("Workflow.List failed, productName:%s, serviceName:%s, serviceType:%s, err:%v", productName, serviceName, serviceType, err) @@ -157,7 +156,7 @@ func ListServiceWorkflows(productName, envName, serviceName, serviceType string, ServiceModule: container.Name, } serviceModuleTarget := fmt.Sprintf("%s%s%s%s%s", service.ProductName, SplitSymbol, service.ServiceName, SplitSymbol, container.Name) - moBuild := findModuleByTargetAndVersion(allModules, serviceModuleTarget) + moBuild, _ := findModuleByTargetAndVersion(allModules, serviceModuleTarget) if moBuild == nil { continue } @@ -170,7 +169,7 @@ func ListServiceWorkflows(productName, envName, serviceName, serviceType string, ServiceModule: service.ServiceName, } serviceModuleTarget := fmt.Sprintf("%s%s%s%s%s", service.ProductName, SplitSymbol, service.ServiceName, SplitSymbol, service.ServiceName) - moBuild := findModuleByTargetAndVersion(allModules, serviceModuleTarget) + moBuild, _ := findModuleByTargetAndVersion(allModules, serviceModuleTarget) if moBuild != nil { resp.Targets = append(resp.Targets, target) } @@ -180,6 +179,7 @@ func ListServiceWorkflows(productName, envName, serviceName, serviceType string, } type CreateTaskResp struct { + ProjectName string `json:"project_name"` PipelineName string `json:"pipeline_name"` TaskID int64 `json:"task_id"` } @@ -215,12 +215,17 @@ func CreateServiceTask(args *commonmodels.ServiceTaskArgs, log *zap.SugaredLogge } stages := make([]*commonmodels.Stage, 0) - subTasks, err := BuildModuleToSubTasks("", args.ServiceName, args.ServiceName, args.ProductName, nil, nil, log) + buildModuleArgs := &commonmodels.BuildModuleArgs{ + Target: args.ServiceName, + ServiceName: args.ServiceName, + ProductName: args.ProductName, + } + subTasks, err := BuildModuleToSubTasks(buildModuleArgs, log) if err != nil { return nil, e.ErrCreateTask.AddErr(err) } - task := &task.Task{ + task := &taskmodels.Task{ Type: config.ServiceType, ProductName: args.ProductName, TaskCreator: args.ServiceTaskCreator, @@ -232,7 +237,9 @@ func CreateServiceTask(args *commonmodels.ServiceTaskArgs, log *zap.SugaredLogge } sort.Sort(ByTaskKind(task.SubTasks)) - if err := ensurePipelineTask(task, "", log); err != nil { + if err := ensurePipelineTask(&taskmodels.TaskOpt{ + Task: task, + }, log); err != nil { log.Errorf("CreateServiceTask ensurePipelineTask err : %v", err) return nil, err } @@ -279,7 +286,7 @@ func CreateServiceTask(args *commonmodels.ServiceTaskArgs, log *zap.SugaredLogge return nil, e.ErrCreateTask } - createTaskResps = append(createTaskResps, &CreateTaskResp{PipelineName: pipelineName, TaskID: nextTaskID}) + createTaskResps = append(createTaskResps, &CreateTaskResp{ProjectName: product.ProductName, PipelineName: pipelineName, TaskID: nextTaskID}) } return createTaskResps, nil @@ -296,6 +303,6 @@ func serviceTaskArgsToTaskArgs(serviceTaskArgs *commonmodels.ServiceTaskArgs) *c resp.Builds = make([]*types.Repository, 0) return resp } - resp.Builds = buildObj.Repos + resp.Builds = commonservice.FindReposByTarget(serviceTaskArgs.ProductName, serviceTaskArgs.ServiceName, serviceTaskArgs.ServiceName, buildObj) return resp } diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/sort.go b/pkg/microservice/aslan/core/workflow/service/workflow/sort.go index 1d6313ac9bbd71692271f83854eeea91f2e17bb8..5e0adc352e9056f3cd68eccc3fa5b0d02ac681d7 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/sort.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/sort.go @@ -36,16 +36,21 @@ var SubtaskOrder = map[config.TaskType]int{ config.TaskType("jira"): 1, config.TaskType("pipeline"): 2, config.TaskType("buildv2"): 3, - config.TaskType("jenkins_build"): 4, - config.TaskType("docker_build"): 5, - config.TaskType("archive"): 6, - config.TaskType("artifact"): 7, - config.TaskType("deploy"): 8, - config.TaskType("testingv2"): 9, - config.TaskType("security"): 10, - config.TaskType("distribute2kodo"): 11, - config.TaskType("release_image"): 12, - config.TaskType("reset_image"): 13, + config.TaskType("buildv3"): 4, + config.TaskType("jenkins_build"): 5, + config.TaskType("docker_build"): 6, + config.TaskType("archive"): 7, + config.TaskType("artifact"): 8, + config.TaskType("artifact_deploy"): 9, + config.TaskType("deploy"): 10, + config.TaskType("testingv2"): 11, + config.TaskType("security"): 12, + config.TaskType("distribute2kodo"): 13, + config.TaskType("release_image"): 14, + config.TaskType("reset_image"): 15, + config.TaskType("trigger"): 16, + config.TaskType("extension"): 17, + config.TaskType("scanning"): 18, } type ByStageKind []*commonmodels.Stage diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/types.go b/pkg/microservice/aslan/core/workflow/service/workflow/types.go new file mode 100644 index 0000000000000000000000000000000000000000..cc30372bbf51498e85082eec7728777ea80d174a --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/service/workflow/types.go @@ -0,0 +1,82 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package workflow + +type WorkflowV3 struct { + ID string `json:"id"` + Name string `json:"name"` + ProjectName string `json:"project_name"` + Description string `json:"description"` + Parameters []*ParameterSetting `json:"parameters"` + SubTasks []map[string]interface{} `json:"sub_tasks"` +} + +type WorkflowV3Brief struct { + ID string `json:"id"` + Name string `json:"name"` + ProjectName string `json:"project_name"` +} + +type ParameterSettingType string + +const ( + StringType ParameterSettingType = "string" + ChoiceType ParameterSettingType = "choice" + ExternalType ParameterSettingType = "external" +) + +type ParameterSetting struct { + // External type parameter will NOT use this key. + Key string `json:"key"` + Type ParameterSettingType `json:"type"` + //DefaultValue is the + DefaultValue string `json:"default_value"` + // choiceOption Are all options enumerated + ChoiceOption []string `json:"choice_option"` + // ExternalSetting It is the configuration of the external system to obtain the variable + ExternalSetting *ExternalSetting `json:"external_setting"` +} + +type ExternalSetting struct { + SystemID string `json:"system_id"` + Endpoint string `json:"endpoint"` + Method string `json:"method"` + Headers []*KV `json:"headers"` + Body string `json:"body"` + Params []*ExternalParamMapping `json:"params"` +} + +type KV struct { + Key string `json:"key"` + Value string `json:"value"` +} + +type ExternalParamMapping struct { + // zadig变量名称 + ParamKey string `json:"param_key"` + // 返回中的key的位置 + ResponseKey string `json:"response_key"` + Display bool `json:"display"` +} + +type WorkflowV3TaskArgs struct { + Type string `json:"type"` + Key string `json:"key,omitempty"` + Value string `json:"value,omitempty"` + Choice []string `json:"choice,omitempty"` + Options []map[string]interface{} `json:"options,omitempty"` +} diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/workflow.go b/pkg/microservice/aslan/core/workflow/service/workflow/workflow.go index ec44581e70f75de144b4c1c55f2cecd4b670d1d0..d3e51cd14400d1546c8624f2d4cd3aa0dfd69990 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/workflow.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/workflow.go @@ -18,7 +18,6 @@ package workflow import ( "fmt" - "strings" "sync" "github.com/hashicorp/go-multierror" @@ -31,12 +30,11 @@ import ( commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/collaboration" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/webhook" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/poetry" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/types" - "github.com/koderover/zadig/pkg/types/permission" ) var mut sync.Mutex @@ -47,6 +45,72 @@ type EnvStatus struct { ErrMessage string `json:"err_message"` } +type Workflow struct { + Name string `json:"name"` + ProjectName string `json:"projectName"` + UpdateTime int64 `json:"updateTime"` + CreateTime int64 `json:"createTime"` + UpdateBy string `json:"updateBy,omitempty"` + Schedules *commonmodels.ScheduleCtrl `json:"schedules,omitempty"` + SchedulerEnabled bool `json:"schedulerEnabled"` + EnabledStages []string `json:"enabledStages"` + IsFavorite bool `json:"isFavorite"` + WorkflowType string `json:"workflow_type"` + RecentTask *TaskInfo `json:"recentTask"` + RecentSuccessfulTask *TaskInfo `json:"recentSuccessfulTask"` + RecentFailedTask *TaskInfo `json:"recentFailedTask"` + AverageExecutionTime float64 `json:"averageExecutionTime"` + SuccessRate float64 `json:"successRate"` + Description string `json:"description,omitempty"` + BaseName string `json:"base_name"` + BaseRefs []string `json:"base_refs"` +} + +type TaskInfo struct { + TaskID int64 `json:"taskID"` + PipelineName string `json:"pipelineName"` + Status string `json:"status"` +} + +type workflowCreateArg struct { + name string + envName string + buildStageEnabled bool + ArtifactStageEnabled bool +} + +type workflowCreateArgs struct { + productName string + argsMap map[string]*workflowCreateArg +} + +func (args *workflowCreateArgs) addWorkflowArg(envName string, buildStageEnabled, artifactStageEnabled bool) { + wName := fmt.Sprintf("%s-workflow-%s", args.productName, envName) + if artifactStageEnabled { + wName = fmt.Sprintf("%s-%s-workflow", args.productName, "ops") + } + // The hosting env workflow name is not bound to the environment + if !artifactStageEnabled && envName == "" { + wName = fmt.Sprintf("%s-workflow", args.productName) + } + args.argsMap[wName] = &workflowCreateArg{ + name: wName, + envName: envName, + buildStageEnabled: buildStageEnabled, + ArtifactStageEnabled: artifactStageEnabled, + } +} + +func (args *workflowCreateArgs) initDefaultWorkflows() { + args.addWorkflowArg("dev", true, false) + args.addWorkflowArg("qa", true, false) + args.addWorkflowArg("", false, true) +} + +func (args *workflowCreateArgs) clear() { + args.argsMap = make(map[string]*workflowCreateArg) +} + func AutoCreateWorkflow(productName string, log *zap.SugaredLogger) *EnvStatus { productTmpl, err := template.NewProductColl().Find(productName) if err != nil { @@ -60,22 +124,42 @@ func AutoCreateWorkflow(productName string, log *zap.SugaredLogger) *EnvStatus { mut.Unlock() }() - workflowNames := []string{productName + "-workflow-dev", productName + "-workflow-qa", productName + "-workflow-ops"} - // 云主机场景不创建ops工作流 - if productTmpl.ProductFeature != nil && productTmpl.ProductFeature.BasicFacility == "cloud_host" { - workflowNames = []string{productName + "-workflow-dev", productName + "-workflow-qa"} - } else if productTmpl.ProductFeature != nil && productTmpl.ProductFeature.CreateEnvType == setting.SourceFromExternal { - workflowNames = []string{productName + "-workflow-dev"} + createArgs := &workflowCreateArgs{ + productName: productName, + argsMap: make(map[string]*workflowCreateArg), + } + createArgs.initDefaultWorkflows() + + // helm project may have customized products, use the real created products + if productTmpl.ProductFeature != nil && productTmpl.ProductFeature.DeployType == setting.HelmDeployType { + productList, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{ + Name: productName, + }) + if err != nil { + log.Errorf("fialed to list products, projectName %s, err %s", productName, err) + } + createArgs.clear() + for _, product := range productList { + createArgs.addWorkflowArg(product.EnvName, true, false) + } + createArgs.addWorkflowArg("", false, true) + } + + // Only one workflow is created in the hosting environment + if productTmpl.ProductFeature != nil && productTmpl.ProductFeature.CreateEnvType == setting.SourceFromExternal { + createArgs.clear() + createArgs.addWorkflowArg("", true, false) } workflowSlice := sets.NewString() - for _, workflowName := range workflowNames { + for workflowName := range createArgs.argsMap { _, err := FindWorkflow(workflowName, log) if err == nil { workflowSlice.Insert(workflowName) } } - if len(workflowSlice) < len(workflowNames) { + + if len(workflowSlice) < len(createArgs.argsMap) { preSetResps, err := PreSetWorkflow(productName, log) if err != nil { errList = multierror.Append(errList, err) @@ -95,12 +179,12 @@ func AutoCreateWorkflow(productName string, log *zap.SugaredLogger) *EnvStatus { artifactModules = append(artifactModules, artifactModule) } - for _, workflowName := range workflowNames { + for workflowName, workflowArg := range createArgs.argsMap { if workflowSlice.Has(workflowName) { continue } - if _, err := commonrepo.NewWorkflowColl().Find(workflowName); err == nil { - errList = multierror.Append(errList, fmt.Errorf("workflow [%s] 在项目 [%s] 中已经存在", workflowName, productName)) + if dupWorkflow, err := commonrepo.NewWorkflowColl().Find(workflowName); err == nil { + errList = multierror.Append(errList, fmt.Errorf("workflow [%s] 在项目 [%s] 中已经存在", workflowName, dupWorkflow.ProductTmplName)) } workflow := new(commonmodels.Workflow) workflow.Enabled = true @@ -108,24 +192,19 @@ func AutoCreateWorkflow(productName string, log *zap.SugaredLogger) *EnvStatus { workflow.Name = workflowName workflow.CreateBy = setting.SystemUser workflow.UpdateBy = setting.SystemUser - workflow.EnvName = "dev" + workflow.EnvName = workflowArg.envName workflow.BuildStage = &commonmodels.BuildStage{ - Enabled: true, + Enabled: workflowArg.buildStageEnabled, Modules: buildModules, } - if strings.Contains(workflowName, "qa") { - workflow.EnvName = "qa" - } - - if strings.Contains(workflowName, "ops") { - //如果是开启artifactStage，则关闭buildStage - workflow.BuildStage.Enabled = false + //如果是开启artifactStage，则关闭buildStage + if workflowArg.ArtifactStageEnabled { workflow.ArtifactStage = &commonmodels.ArtifactStage{ Enabled: true, Modules: artifactModules, } - workflow.EnvName = "ops" + workflow.EnvName = "ops" //TODO necessary to set a fake env name? } workflow.Schedules = &commonmodels.ScheduleCtrl{ @@ -162,7 +241,7 @@ func AutoCreateWorkflow(productName string, log *zap.SugaredLogger) *EnvStatus { return &EnvStatus{Status: setting.ProductStatusFailed, ErrMessage: err.Error()} } return &EnvStatus{Status: setting.ProductStatusCreating} - } else if len(workflowSlice) == len(workflowNames) { + } else if len(workflowSlice) == len(createArgs.argsMap) { return &EnvStatus{Status: setting.ProductStatusSuccess} } return nil @@ -206,9 +285,77 @@ func FindWorkflow(workflowName string, log *zap.SugaredLogger) (*commonmodels.Wo } resp.Schedules = &schedule } + if resp.BuildStage.Enabled { + // make a map of current target modules + buildMap := map[string]bool{} + + moList, err := commonrepo.NewBuildColl().List(&commonrepo.BuildListOption{}) + if err != nil { + return resp, e.ErrListTemplate.AddDesc(err.Error()) + } + for _, build := range resp.BuildStage.Modules { + key := fmt.Sprintf("%s-%s-%s", build.Target.ProductName, build.Target.ServiceName, build.Target.ServiceModule) + buildMap[key] = true + if build.Target.BuildName == "" { + build.Target.BuildName = findBuildName(key, moList) + } + } + + services, err := commonrepo.NewServiceColl().ListMaxRevisionsByProduct(resp.ProductTmplName) + if err != nil { + log.Errorf("ServiceTmpl.ListMaxRevisions error: %v", err) + return resp, e.ErrListTemplate.AddDesc(err.Error()) + } + + for _, serviceTmpl := range services { + switch serviceTmpl.Type { + case setting.PMDeployType: + // PM service does not have such logic + break + + case setting.K8SDeployType, setting.HelmDeployType: + for _, container := range serviceTmpl.Containers { + key := fmt.Sprintf("%s-%s-%s", serviceTmpl.ProductName, serviceTmpl.ServiceName, container.Name) + // if no target info is found for this container, meaning that this is a new service for that workflow + // then we need to add it to the response + if _, ok := buildMap[key]; !ok { + resp.BuildStage.Modules = append(resp.BuildStage.Modules, &commonmodels.BuildModule{ + HideServiceModule: false, + BuildModuleVer: "stable", + Target: &commonmodels.ServiceModuleTarget{ + ProductName: serviceTmpl.ProductName, + ServiceName: serviceTmpl.ServiceName, + ServiceModule: container.Name, + BuildName: findBuildName(key, moList), + }, + }) + } + } + } + } + } + + for _, module := range resp.BuildStage.Modules { + for _, bf := range module.BranchFilter { + bf.RepoNamespace = bf.GetNamespace() + } + } + return resp, nil } +func findBuildName(key string, moList []*commonmodels.Build) string { + for _, mo := range moList { + for _, moTarget := range mo.Targets { + moduleTargetStr := fmt.Sprintf("%s-%s-%s", moTarget.ProductName, moTarget.ServiceName, moTarget.ServiceModule) + if key == moduleTargetStr { + return mo.Name + } + } + } + return "" +} + type PreSetResp struct { Target *commonmodels.ServiceModuleTarget `json:"target"` BuildModuleVers []string `json:"build_module_vers"` @@ -262,50 +409,42 @@ func PreSetWorkflow(productName string, log *zap.SugaredLogger) ([]*PreSetResp, log.Errorf("[Build.List] error: %v", err) return nil, e.ErrListBuildModule.AddErr(err) } - for k, v := range targets { - // 选择了一个特殊字符在项目名称、服务名称以及服务组件名称里面都不允许的特殊字符，避免出现异常 - targetArr := strings.Split(k, SplitSymbol) - if len(targetArr) != 3 { - continue - } - - preSet := &PreSetResp{ - Target: &commonmodels.ServiceModuleTarget{ - ProductName: targetArr[0], - ServiceName: targetArr[1], - ServiceModule: targetArr[2], - }, - Deploy: v, - BuildModuleVers: []string{}, - Repos: make([]*types.Repository, 0), - } - - for _, mo := range moList { - for _, moTarget := range mo.Targets { - moduleTargetStr := fmt.Sprintf("%s%s%s%s%s", moTarget.ProductName, SplitSymbol, moTarget.ServiceName, SplitSymbol, moTarget.ServiceModule) - if moduleTargetStr == k { - if len(mo.Repos) == 0 { - preSet.Repos = make([]*types.Repository, 0) - } else { - preSet.Repos = mo.Repos - } + for _, mo := range moList { + for _, moTarget := range mo.Targets { + target := fmt.Sprintf("%s%s%s%s%s", moTarget.ProductName, SplitSymbol, moTarget.ServiceName, SplitSymbol, moTarget.ServiceModule) + if v, ok := targets[target]; ok { + preSet := &PreSetResp{ + Target: &commonmodels.ServiceModuleTarget{ + ProductName: moTarget.ProductName, + ServiceName: moTarget.ServiceName, + ServiceModule: moTarget.ServiceModule, + BuildName: mo.Name, + }, + Deploy: v, + BuildModuleVers: []string{}, + Repos: make([]*types.Repository, 0), } + if mo.TemplateID != "" { + preSet.Repos = moTarget.Repos + } else { + preSet.Repos = mo.SafeRepos() + } + resp = append(resp, preSet) } } - resp = append(resp, preSet) } return resp, nil } func CreateWorkflow(workflow *commonmodels.Workflow, log *zap.SugaredLogger) error { - _, err := commonrepo.NewWorkflowColl().Find(workflow.Name) + existedWorkflow, err := commonrepo.NewWorkflowColl().Find(workflow.Name) if err == nil { - errStr := fmt.Sprintf("workflow [%s] 在项目 [%s] 中已经存在!", workflow.Name, workflow.ProductTmplName) + errStr := fmt.Sprintf("workflow [%s] 在项目 [%s] 中已经存在!", workflow.Name, existedWorkflow.ProductTmplName) return e.ErrUpsertWorkflow.AddDesc(errStr) } if !checkWorkflowSubModule(workflow) { - return e.ErrUpsertWorkflow.AddDesc("workflow中没有子模块，请设置子模块") + return e.ErrUpsertWorkflow.AddDesc("未检测到构建部署或交付物部署，请配置一项") } if err := validateWorkflowHookNames(workflow); err != nil { @@ -367,7 +506,7 @@ func checkWorkflowSubModule(workflow *commonmodels.Workflow) bool { func UpdateWorkflow(workflow *commonmodels.Workflow, log *zap.SugaredLogger) error { if !checkWorkflowSubModule(workflow) { - return e.ErrUpsertWorkflow.AddDesc("workflow中没有子模块，请设置子模块") + return e.ErrUpsertWorkflow.AddDesc("未检测到构建部署或交付物部署，请配置一项") } currentWorkflow, err := commonrepo.NewWorkflowColl().Find(workflow.Name) @@ -424,117 +563,150 @@ func validateWorkflowHookNames(w *commonmodels.Workflow) error { return validateHookNames(names) } -func ListWorkflows(queryType string, userID int, log *zap.SugaredLogger) ([]*commonmodels.Workflow, error) { - workflows, err := commonrepo.NewWorkflowColl().List(&commonrepo.ListWorkflowOption{}) +func ListWorkflows(projects []string, userID string, names []string, log *zap.SugaredLogger) ([]*Workflow, error) { + existingProjects, err := template.NewProductColl().ListNames(projects) if err != nil { - log.Errorf("Workflow.List error: %v", err) - return workflows, e.ErrListWorkflow.AddDesc(err.Error()) + log.Errorf("Failed to list projects, err: %s", err) + return nil, e.ErrListWorkflow.AddDesc(err.Error()) } - favorites, err := commonrepo.NewFavoriteColl().List(&commonrepo.FavoriteArgs{UserID: userID, Type: string(config.WorkflowType)}) + workflows, err := commonrepo.NewWorkflowColl().List(&commonrepo.ListWorkflowOption{Projects: existingProjects, Names: names}) if err != nil { - log.Errorf("list favorite error: %v", err) - return workflows, e.ErrListFavorite + log.Errorf("Failed to list workflows, err: %s", err) + return nil, e.ErrListWorkflow.AddDesc(err.Error()) } - workflowStats, err := commonrepo.NewWorkflowStatColl().FindWorkflowStat(&commonrepo.WorkflowStatArgs{Type: string(config.WorkflowType)}) + var workflowNames []string + var res []*Workflow + workflowCMMap, err := collaboration.GetWorkflowCMMap(projects, log) if err != nil { - log.Errorf("list workflow stat error: %v", err) - return workflows, fmt.Errorf("列出工作流统计失败") + return nil, err } - - for _, workflow := range workflows { - if queryType == "artifact" { - if workflow.ArtifactStage == nil || !workflow.ArtifactStage.Enabled { - continue - } + for _, w := range workflows { + stages := make([]string, 0, 4) + if w.BuildStage != nil && w.BuildStage.Enabled { + stages = append(stages, "build") } - latestTask, _ := commonrepo.NewTaskColl().FindLatestTask(&commonrepo.FindTaskOption{PipelineName: workflow.Name, Type: config.WorkflowType}) - if latestTask != nil { - workflow.LastestTask = &commonmodels.TaskInfo{TaskID: latestTask.TaskID, PipelineName: latestTask.PipelineName, Status: latestTask.Status} + if w.ArtifactStage != nil && w.ArtifactStage.Enabled { + stages = append(stages, "artifact") } - - latestPassedTask, _ := commonrepo.NewTaskColl().FindLatestTask(&commonrepo.FindTaskOption{PipelineName: workflow.Name, Type: config.WorkflowType, Status: config.StatusPassed}) - if latestPassedTask != nil { - workflow.LastSucessTask = &commonmodels.TaskInfo{TaskID: latestPassedTask.TaskID, PipelineName: latestPassedTask.PipelineName} + if w.TestStage != nil && w.TestStage.Enabled { + stages = append(stages, "test") } - - latestFailedTask, _ := commonrepo.NewTaskColl().FindLatestTask(&commonrepo.FindTaskOption{PipelineName: workflow.Name, Type: config.WorkflowType, Status: config.StatusFailed}) - if latestFailedTask != nil { - workflow.LastFailureTask = &commonmodels.TaskInfo{TaskID: latestFailedTask.TaskID, PipelineName: latestFailedTask.PipelineName} + if w.DistributeStage != nil && w.DistributeStage.Enabled { + stages = append(stages, "distribute") } + var baseRefs []string + if cmSet, ok := workflowCMMap[collaboration.BuildWorkflowCMMapKey(w.ProductTmplName, w.Name)]; ok { + for _, cm := range cmSet.List() { + baseRefs = append(baseRefs, cm) + } + } + res = append(res, &Workflow{ + Name: w.Name, + ProjectName: w.ProductTmplName, + UpdateTime: w.UpdateTime, + CreateTime: w.CreateTime, + UpdateBy: w.UpdateBy, + SchedulerEnabled: w.ScheduleEnabled, + Schedules: w.Schedules, + EnabledStages: stages, + Description: w.Description, + BaseName: w.BaseName, + BaseRefs: baseRefs, + }) - workflow.IsFavorite = IsFavoriteWorkflow(workflow, favorites) + workflowNames = append(workflowNames, w.Name) + } - workflow.TotalDuration, workflow.TotalNum, workflow.TotalSuccess = findWorkflowStat(workflow, workflowStats) + favorites, err := commonrepo.NewFavoriteColl().List(&commonrepo.FavoriteArgs{UserID: userID, Type: string(config.WorkflowType)}) + if err != nil { + log.Warnf("Failed to list favorites, err: %s", err) + } + favoriteSet := sets.NewString() + for _, f := range favorites { + favoriteSet.Insert(f.Name) } - return workflows, nil -} + workflowStats, err := commonrepo.NewWorkflowStatColl().FindWorkflowStat(&commonrepo.WorkflowStatArgs{Names: workflowNames, Type: string(config.WorkflowType)}) + if err != nil { + log.Warnf("Failed to list workflow stats, err: %s", err) + } + workflowStatMap := make(map[string]*commonmodels.WorkflowStat) + for _, s := range workflowStats { + workflowStatMap[s.Name] = s + } -func IsFavoriteWorkflow(workflow *commonmodels.Workflow, favorites []*commonmodels.Favorite) bool { - for _, favorite := range favorites { - if workflow.Name == favorite.Name && workflow.ProductTmplName == favorite.ProductName { - return true - } + tasks, err := commonrepo.NewTaskColl().ListPreview(workflowNames) + if err != nil { + log.Warnf("Failed to list workflow tasks, err: %s", err) } - return false -} + for _, r := range res { + getRecentTaskInfo(r, tasks) + + if favoriteSet.Has(r.Name) { + r.IsFavorite = true + } -func findWorkflowStat(workflow *commonmodels.Workflow, workflowStats []*commonmodels.WorkflowStat) (int64, int, int) { - for _, workflowStat := range workflowStats { - if workflow.Name == workflowStat.Name { - return workflowStat.TotalDuration, workflowStat.TotalSuccess + workflowStat.TotalFailure, workflowStat.TotalSuccess + if s, ok := workflowStatMap[r.Name]; ok { + total := float64(s.TotalSuccess + s.TotalFailure) + successful := float64(s.TotalSuccess) + totalDuration := float64(s.TotalDuration) + + r.AverageExecutionTime = totalDuration / total + r.SuccessRate = successful / total } } - return 0, 0, 0 + + return res, nil } -func ListAllWorkflows(testName string, userID int, superUser bool, log *zap.SugaredLogger) ([]*commonmodels.Workflow, error) { - allWorkflows := make([]*commonmodels.Workflow, 0) - workflows := make([]*commonmodels.Workflow, 0) - var err error - if superUser { - allWorkflows, err = commonrepo.NewWorkflowColl().List(&commonrepo.ListWorkflowOption{}) - if err != nil { - log.Errorf("Workflow.List error: %v", err) - return allWorkflows, e.ErrListWorkflow.AddDesc(err.Error()) +func getRecentTaskInfo(workflow *Workflow, tasks []*commonrepo.TaskPreview) { + recentTask := &commonrepo.TaskPreview{} + recentFailedTask := &commonrepo.TaskPreview{} + recentSucceedTask := &commonrepo.TaskPreview{} + for _, task := range tasks { + if task.PipelineName != workflow.Name { + continue } - } else { - poetryCtl := poetry.New(config.PoetryAPIServer(), config.PoetryAPIRootKey()) - productNameMap, err := poetryCtl.GetUserProject(userID, log) - if err != nil { - log.Errorf("ListAllWorkflows GetUserProject error: %v", err) - return nil, fmt.Errorf("ListAllWorkflows GetUserProject error: %v", err) - } - productNames := make([]string, 0) - for productName, roleIDs := range productNameMap { - roleID := roleIDs[0] - if roleID == setting.RoleOwnerID { - productNames = append(productNames, productName) - } else { - uuids, err := poetryCtl.GetUserPermissionUUIDs(roleID, productName, log) - if err != nil { - log.Errorf("ListAllWorkflows GetUserPermissionUUIDs error: %v", err) - return nil, fmt.Errorf("ListAllWorkflows GetUserPermissionUUIDs error: %v", err) - } - - ids := sets.NewString(uuids...) - if ids.Has(permission.WorkflowUpdateUUID) { - productNames = append(productNames, productName) - } - } + if task.TaskID > recentTask.TaskID { + recentTask = task } - for _, productName := range productNames { - tempWorkflows, err := commonrepo.NewWorkflowColl().List(&commonrepo.ListWorkflowOption{ProductName: productName}) - if err != nil { - log.Errorf("ListAllWorkflows Workflow.List error: %v", err) - return nil, fmt.Errorf("ListAllWorkflows Workflow.List error: %v", err) - } - allWorkflows = append(allWorkflows, tempWorkflows...) + if task.Status == config.StatusPassed && task.TaskID > recentSucceedTask.TaskID { + recentSucceedTask = task + } + if task.Status == config.StatusFailed && task.TaskID > recentFailedTask.TaskID { + recentFailedTask = task + } + } + if recentTask.TaskID > 0 { + workflow.RecentTask = &TaskInfo{ + TaskID: recentTask.TaskID, + PipelineName: recentTask.PipelineName, + Status: string(recentTask.Status), + } + } + if recentSucceedTask.TaskID > 0 { + workflow.RecentSuccessfulTask = &TaskInfo{ + TaskID: recentSucceedTask.TaskID, + PipelineName: recentSucceedTask.PipelineName, + Status: string(recentSucceedTask.Status), + } + } + if recentFailedTask.TaskID > 0 { + workflow.RecentFailedTask = &TaskInfo{ + TaskID: recentFailedTask.TaskID, + PipelineName: recentFailedTask.PipelineName, + Status: string(recentFailedTask.Status), } } +} +func ListTestWorkflows(testName string, projects []string, log *zap.SugaredLogger) (workflows []*commonmodels.Workflow, err error) { + allWorkflows, err := commonrepo.NewWorkflowColl().ListWorkflowsByProjects(projects) + if err != nil { + return nil, err + } for _, workflow := range allWorkflows { if workflow.TestStage != nil { testNames := sets.NewString(workflow.TestStage.TestNames...) @@ -569,3 +741,51 @@ func CopyWorkflow(oldWorkflowName, newWorkflowName, username string, log *zap.Su return commonrepo.NewWorkflowColl().Create(oldWorkflow) } + +type WorkflowCopyItem struct { + ProjectName string `json:"project_name"` + Old string `json:"old"` + New string `json:"new"` + BaseName string `json:"base_name"` +} + +type BulkCopyWorkflowArgs struct { + Items []WorkflowCopyItem `json:"items"` +} + +func BulkCopyWorkflow(args BulkCopyWorkflowArgs, username string, log *zap.SugaredLogger) error { + var workflows []commonrepo.Workflow + + for _, item := range args.Items { + workflows = append(workflows, commonrepo.Workflow{ + ProjectName: item.ProjectName, + Name: item.Old, + }) + } + oldWorkflows, err := commonrepo.NewWorkflowColl().ListByWorkflows(commonrepo.ListWorkflowOpt{ + Workflows: workflows, + }) + if err != nil { + log.Error(err) + return e.ErrGetPipeline.AddErr(err) + } + workflowMap := make(map[string]*commonmodels.Workflow) + for _, workflow := range oldWorkflows { + workflowMap[workflow.ProductTmplName+"-"+workflow.Name] = workflow + } + var newWorkflows []*commonmodels.Workflow + for _, workflow := range args.Items { + if item, ok := workflowMap[workflow.ProjectName+"-"+workflow.Old]; ok { + newItem := *item + newItem.UpdateBy = username + newItem.Name = workflow.New + newItem.BaseName = workflow.BaseName + newItem.ID = primitive.NewObjectID() + + newWorkflows = append(newWorkflows, &newItem) + } else { + return fmt.Errorf("workflow:%s not exist", item.ProductTmplName+"-"+item.Name) + } + } + return commonrepo.NewWorkflowColl().BulkCreate(newWorkflows) +} diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/workflow_task.go b/pkg/microservice/aslan/core/workflow/service/workflow/workflow_task.go index 4fe6f552badf9b3d4c42ee82ec5480f606c38d1b..27c6980b1c109debd1a7e9d6cdc1156dc0380d9d 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/workflow_task.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/workflow_task.go @@ -17,14 +17,18 @@ limitations under the License. package workflow import ( + "bytes" "errors" "fmt" + "net/url" "reflect" "regexp" "sort" "strconv" "strings" "sync" + gotempl "text/template" + "time" "go.mongodb.org/mongo-driver/mongo" "go.uber.org/zap" @@ -34,18 +38,20 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" + taskmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" + templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/scmnotify" + templ "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/template" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/poetry" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/types" - "github.com/koderover/zadig/pkg/types/permission" "github.com/koderover/zadig/pkg/util" ) @@ -73,7 +79,7 @@ func GetWorkflowArgs(productName, namespace string, log *zap.SugaredLogger) (*Cr return resp, e.ErrListBuildModule.AddDesc(err.Error()) } - targetMap := getProductTargetMap(product) + targetMap, _ := commonservice.GetProductTargetMap(product) projectTargets := getProjectTargets(product.ProductName) targets := make([]*commonmodels.TargetArgs, 0) for _, container := range projectTargets { @@ -86,29 +92,32 @@ func GetWorkflowArgs(productName, namespace string, log *zap.SugaredLogger) (*Cr } target := &commonmodels.TargetArgs{Name: containerArr[2], ServiceName: containerArr[1], Deploy: targetMap[container], Build: &commonmodels.BuildArgs{}, HasBuild: true} - moBuild := findModuleByTargetAndVersion(allModules, container) + moBuild, _ := findModuleByTargetAndVersion(allModules, container) if moBuild == nil { moBuild = &commonmodels.Build{} target.HasBuild = false } - - if len(moBuild.Repos) == 0 { - target.Build.Repos = make([]*types.Repository, 0) - } else { - target.Build.Repos = moBuild.Repos + err = fillBuildDetail(moBuild, containerArr[1], containerArr[2]) + if err != nil { + return resp, e.ErrListBuildModule.AddErr(err) } + target.Build.Repos = moBuild.SafeRepos() + if moBuild.PreBuild != nil { EnsureBuildResp(moBuild) target.Envs = moBuild.PreBuild.Envs } if moBuild.JenkinsBuild != nil { - jenkinsBuildParams := make([]*commonmodels.JenkinsBuildParam, 0) + jenkinsBuildParams := make([]*types.JenkinsBuildParam, 0) for _, jenkinsBuildParam := range moBuild.JenkinsBuild.JenkinsBuildParam { - jenkinsBuildParams = append(jenkinsBuildParams, &commonmodels.JenkinsBuildParam{ - Name: jenkinsBuildParam.Name, - Value: jenkinsBuildParam.Value, + jenkinsBuildParams = append(jenkinsBuildParams, &types.JenkinsBuildParam{ + Name: jenkinsBuildParam.Name, + Value: jenkinsBuildParam.Value, + Type: jenkinsBuildParam.Type, + ChoiceOption: jenkinsBuildParam.ChoiceOption, + AutoGenerate: jenkinsBuildParam.AutoGenerate, }) } target.JenkinsBuildArgs = &commonmodels.JenkinsBuildArgs{ @@ -123,45 +132,25 @@ func GetWorkflowArgs(productName, namespace string, log *zap.SugaredLogger) (*Cr return resp, nil } -func getProductTargetMap(prod *commonmodels.Product) map[string][]commonmodels.DeployEnv { - resp := make(map[string][]commonmodels.DeployEnv) - if prod.Source == setting.SourceFromExternal { - services, _ := commonrepo.NewServiceColl().ListExternalWorkloadsBy(prod.ProductName, prod.EnvName) - for _, service := range services { - for _, container := range service.Containers { - env := service.ServiceName + "/" + container.Name - deployEnv := commonmodels.DeployEnv{Type: setting.K8SDeployType, Env: env} - target := strings.Join([]string{service.ProductName, service.ServiceName, container.Name}, SplitSymbol) - resp[target] = append(resp[target], deployEnv) +func getHideServiceModules(workflow *commonmodels.Workflow) sets.String { + hideServiceModules := sets.NewString() + if workflow.BuildStage != nil && workflow.BuildStage.Enabled { + for _, buildModule := range workflow.BuildStage.Modules { + if buildModule.HideServiceModule { + hideServiceModules.Insert(strings.Join([]string{buildModule.Target.ProductName, buildModule.Target.ServiceName, buildModule.Target.ServiceModule}, SplitSymbol)) } } - return resp } - for _, services := range prod.Services { - for _, serviceObj := range services { - switch serviceObj.Type { - case setting.K8SDeployType: - for _, container := range serviceObj.Containers { - env := serviceObj.ServiceName + "/" + container.Name - deployEnv := commonmodels.DeployEnv{Type: setting.K8SDeployType, Env: env} - target := strings.Join([]string{serviceObj.ProductName, serviceObj.ServiceName, container.Name}, SplitSymbol) - resp[target] = append(resp[target], deployEnv) - } - case setting.PMDeployType: - deployEnv := commonmodels.DeployEnv{Type: setting.PMDeployType, Env: serviceObj.ServiceName} - target := strings.Join([]string{serviceObj.ProductName, serviceObj.ServiceName, serviceObj.ServiceName}, SplitSymbol) - resp[target] = append(resp[target], deployEnv) - case setting.HelmDeployType: - for _, container := range serviceObj.Containers { - env := serviceObj.ServiceName + "/" + container.Name - deployEnv := commonmodels.DeployEnv{Type: setting.HelmDeployType, Env: env} - target := strings.Join([]string{serviceObj.ProductName, serviceObj.ServiceName, container.Name}, SplitSymbol) - resp[target] = append(resp[target], deployEnv) - } + + if workflow.ArtifactStage != nil && workflow.ArtifactStage.Enabled { + for _, artifactModule := range workflow.ArtifactStage.Modules { + if artifactModule.HideServiceModule { + hideServiceModules.Insert(strings.Join([]string{artifactModule.Target.ProductName, artifactModule.Target.ServiceName, artifactModule.Target.ServiceModule}, SplitSymbol)) } } } - return resp + + return hideServiceModules } func getProjectTargets(productName string) []string { @@ -191,10 +180,85 @@ func getProjectTargets(productName string) []string { return targets } -func findModuleByTargetAndVersion(allModules []*commonmodels.Build, serviceModuleTarget string) *commonmodels.Build { +func findModuleByContainer(productName, serviceModuleTarget string, buildStageModules []*commonmodels.BuildModule, allModules []*commonmodels.Build, log *zap.SugaredLogger) (*commonmodels.Build, *commonmodels.ServiceModuleTarget) { + // find build name + var buildName string + for _, buildStageModule := range buildStageModules { + if buildStageModule.Target == nil { + return nil, nil + } + targetStr := fmt.Sprintf("%s%s%s%s%s", buildStageModule.Target.ProductName, SplitSymbol, buildStageModule.Target.ServiceName, SplitSymbol, buildStageModule.Target.ServiceModule) + if serviceModuleTarget == targetStr { + buildName = buildStageModule.Target.BuildName + break + } + } + if buildName == "" { + // Compatible with old data if buildName is empty + productTmpl, err := template.NewProductColl().Find(productName) + if err != nil { + log.Errorf("failed to find project,err:%s", err) + return nil, nil + } + services, err := commonrepo.NewServiceColl().ListMaxRevisionsForServices(productTmpl.AllServiceInfos(), "") + if err != nil { + log.Errorf("failed to list services,err:%s", err) + return nil, nil + } + + for _, serviceTmpl := range services { + if serviceTmpl.Type != setting.PMDeployType { + for _, container := range serviceTmpl.Containers { + targetStr := fmt.Sprintf("%s%s%s%s%s", serviceTmpl.ProductName, SplitSymbol, serviceTmpl.ServiceName, SplitSymbol, container.Name) + if serviceModuleTarget == targetStr { + buildName = findBuildNameByContainerName(container.Name, serviceTmpl) + } + } + } else if serviceTmpl.Type == setting.PMDeployType { + targetStr := fmt.Sprintf("%s%s%s%s%s", serviceTmpl.ProductName, SplitSymbol, serviceTmpl.ServiceName, SplitSymbol, serviceTmpl.ServiceName) + if serviceModuleTarget == targetStr { + buildName = findBuildNameByContainerName(serviceTmpl.ServiceName, serviceTmpl) + } + } + } + } + if buildName != "" { + for _, module := range allModules { + if module.Name == buildName { + for _, target := range module.Targets { + targetStr := fmt.Sprintf("%s%s%s%s%s", target.ProductName, SplitSymbol, target.ServiceName, SplitSymbol, target.ServiceModule) + if targetStr == serviceModuleTarget { + return module, target + } + } + } + } + } + + return nil, nil +} + +func findBuildNameByContainerName(containerName string, serviceTmpl *commonmodels.Service) string { + opt := &commonrepo.BuildListOption{ + ServiceName: serviceTmpl.ServiceName, + Targets: []string{containerName}, + } + if serviceTmpl.Visibility != setting.PublicService { + opt.ProductName = serviceTmpl.ProductName + } + + buildModules, err := commonrepo.NewBuildColl().List(opt) + if err != nil || len(buildModules) == 0 { + return "" + } + buildName := buildModules[0].Name + return buildName +} + +func findModuleByTargetAndVersion(allModules []*commonmodels.Build, serviceModuleTarget string) (*commonmodels.Build, *commonmodels.ServiceModuleTarget) { containerArr := strings.Split(serviceModuleTarget, SplitSymbol) if len(containerArr) != 3 { - return nil + return nil, nil } opt := &commonrepo.ServiceFindOption{ @@ -210,11 +274,11 @@ func findModuleByTargetAndVersion(allModules []*commonmodels.Build, serviceModul for _, target := range mo.Targets { targetStr := fmt.Sprintf("%s%s%s%s%s", target.ProductName, SplitSymbol, target.ServiceName, SplitSymbol, target.ServiceModule) if targetStr == strings.Join(containerArr, SplitSymbol) { - return mo + return mo, target } } } - return nil + return nil, nil } func EnsureBuildResp(mb *commonmodels.Build) { @@ -222,8 +286,9 @@ func EnsureBuildResp(mb *commonmodels.Build) { mb.Targets = make([]*commonmodels.ServiceModuleTarget, 0) } - if len(mb.Repos) == 0 { - mb.Repos = make([]*types.Repository, 0) + mb.Repos = mb.SafeRepos() + for _, repo := range mb.Repos { + repo.RepoNamespace = repo.GetRepoNamespace() } if mb.PreBuild != nil { @@ -274,6 +339,13 @@ func PresetWorkflowArgs(namespace, workflowName string, log *zap.SugaredLogger) log.Errorf("Workflow.Find error: %v", err) return resp, e.ErrFindWorkflow.AddDesc(err.Error()) } + filtermap := make(map[string][]*types.BranchFilterInfo) + if workflow.BuildStage.Enabled { + for _, buildModule := range workflow.BuildStage.Modules { + buildKey := fmt.Sprintf("%s/%s", buildModule.Target.ServiceModule, buildModule.Target.ServiceName) + filtermap[buildKey] = buildModule.BranchFilter + } + } if workflow.DistributeStage != nil { resp.DistributeEnabled = workflow.DistributeStage.Enabled } @@ -297,11 +369,15 @@ func PresetWorkflowArgs(namespace, workflowName string, log *zap.SugaredLogger) return resp, e.ErrListTestModule.AddDesc(err.Error()) } - targetMap := getProductTargetMap(product) + targetMap, imageNameM := commonservice.GetProductTargetMap(product) projectTargets := getProjectTargets(product.ProductName) + hideServiceModules := getHideServiceModules(workflow) targets := make([]*commonmodels.TargetArgs, 0) if (workflow.BuildStage != nil && workflow.BuildStage.Enabled) || (workflow.ArtifactStage != nil && workflow.ArtifactStage.Enabled) { for _, container := range projectTargets { + if hideServiceModules.Has(container) { + continue + } if _, ok := targetMap[container]; !ok { continue } @@ -315,19 +391,26 @@ func PresetWorkflowArgs(namespace, workflowName string, log *zap.SugaredLogger) ServiceName: containerArr[1], ProductName: containerArr[0], Deploy: targetMap[container], + ImageName: imageNameM[container], Build: &commonmodels.BuildArgs{}, HasBuild: true, } - moBuild := findModuleByTargetAndVersion(allModules, container) + + moBuild, targetInfo := findModuleByContainer(workflow.ProductTmplName, container, workflow.BuildStage.Modules, allModules, log) if moBuild == nil { moBuild = &commonmodels.Build{} target.HasBuild = false } + target.BuildName = moBuild.Name + err = fillBuildDetail(moBuild, containerArr[1], containerArr[2]) + if err != nil { + return resp, e.ErrListBuildModule.AddErr(err) + } - if len(moBuild.Repos) == 0 { - target.Build.Repos = make([]*types.Repository, 0) + if moBuild.TemplateID != "" { + target.Build.Repos = targetInfo.Repos } else { - target.Build.Repos = moBuild.Repos + target.Build.Repos = moBuild.SafeReposDeepCopy() } if moBuild.PreBuild != nil { @@ -336,11 +419,14 @@ func PresetWorkflowArgs(namespace, workflowName string, log *zap.SugaredLogger) } if moBuild.JenkinsBuild != nil { - jenkinsBuildParams := make([]*commonmodels.JenkinsBuildParam, 0) + jenkinsBuildParams := make([]*types.JenkinsBuildParam, 0) for _, jenkinsBuildParam := range moBuild.JenkinsBuild.JenkinsBuildParam { - jenkinsBuildParams = append(jenkinsBuildParams, &commonmodels.JenkinsBuildParam{ - Name: jenkinsBuildParam.Name, - Value: jenkinsBuildParam.Value, + jenkinsBuildParams = append(jenkinsBuildParams, &types.JenkinsBuildParam{ + Name: jenkinsBuildParam.Name, + Value: jenkinsBuildParam.Value, + Type: jenkinsBuildParam.Type, + ChoiceOption: jenkinsBuildParam.ChoiceOption, + AutoGenerate: jenkinsBuildParam.AutoGenerate, }) } target.JenkinsBuildArgs = &commonmodels.JenkinsBuildArgs{ @@ -354,6 +440,21 @@ func PresetWorkflowArgs(namespace, workflowName string, log *zap.SugaredLogger) } } + for _, target := range targets { + key := fmt.Sprintf("%s/%s", target.Name, target.ServiceName) + for _, repoInfo := range target.Build.Repos { + if filterList, ok := filtermap[key]; ok { + for _, filter := range filterList { + // make sure they are the same repository + if filter.CodehostID == repoInfo.CodehostID && filter.RepoOwner == repoInfo.RepoOwner && filter.RepoName == repoInfo.RepoName { + repoInfo.FilterRegexp = filter.FilterRegExp + break + } + } + } + } + } + resp.Target = targets testArgs := make([]*commonmodels.TestArgs, 0) testsMap := make(map[string]*commonmodels.Testing) @@ -381,6 +482,8 @@ func PresetWorkflowArgs(namespace, workflowName string, log *zap.SugaredLogger) Key: moduleEnv.Key, Value: moduleEnv.Value, IsCredential: moduleEnv.IsCredential, + ChoiceOption: moduleEnv.ChoiceOption, + Type: moduleEnv.Type, }) } } @@ -412,38 +515,42 @@ func PresetWorkflowArgs(namespace, workflowName string, log *zap.SugaredLogger) return resp, nil } -func CreateWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator string, userID int, superUser bool, log *zap.SugaredLogger) (*CreateTaskResp, error) { +func CreateWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator string, log *zap.SugaredLogger) (*CreateTaskResp, error) { if args == nil { return nil, fmt.Errorf("args should not be nil") } - // RequestMode=openAPI表示外部客户调用API，部分数据需要获取并补充到args中 - if args.RequestMode == "openAPI" { + // RequestMode=openAPI means that external clients call the API, and some data needs to be obtained and added to args + if args.RequestMode == setting.RequestModeOpenAPI { log.Info("CreateWorkflowTask from openAPI") - err := AddDataToArgs(args, log) + resp, err := AddDataToArgsOrCreateReleaseImageTask(args, log) if err != nil { log.Errorf("AddDataToArgs error: %v", err) return nil, err } + // create release image task + if resp != nil { + return resp, nil + } + taskCreator = setting.RequestModeOpenAPI } - productTempl, err := template.NewProductColl().Find(args.ProductTmplName) + workflow, err := commonrepo.NewWorkflowColl().Find(args.WorkflowName) if err != nil { - log.Errorf("productTempl.Find error: %v", err) + log.Errorf("Workflow.Find error: %v", err) return nil, e.ErrFindWorkflow.AddDesc(err.Error()) } - workflow, err := commonrepo.NewWorkflowColl().Find(args.WorkflowName) + project, err := template.NewProductColl().Find(workflow.ProductTmplName) if err != nil { - log.Errorf("Workflow.Find error: %v", err) + log.Errorf("project.Find error: %v", err) return nil, e.ErrFindWorkflow.AddDesc(err.Error()) } - args.IsParallel = workflow.IsParallel - - if !HasPermission(workflow.ProductTmplName, workflow.EnvName, args, userID, superUser, log) { - log.Warnf("该工作流[%s]绑定的环境您没有权限,用户[%s]不能执行该工作流!", workflow.Name, taskCreator) - return nil, e.ErrCreateTask.AddDesc("该工作流绑定的环境您没有更新环境或者环境管理权限,不能执行该工作流!") + // developer don't pass args.ProductTmplName + if args.ProductTmplName == "" { + args.ProductTmplName = workflow.ProductTmplName } + args.IsParallel = workflow.IsParallel var env *commonmodels.Product if args.Namespace != "" { @@ -467,31 +574,29 @@ func CreateWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator string, } } - nextTaskID, err := commonrepo.NewCounterColl().GetNextSeq(fmt.Sprintf(setting.WorkflowTaskFmt, args.WorkflowName)) - if err != nil { - log.Errorf("Counter.GetNextSeq error: %v", err) - return nil, e.ErrGetCounter.AddDesc(err.Error()) - } - - // 获取全局configpayload + // get global configPayload configPayload := commonservice.GetConfigPayload(args.CodehostID) - repos, err := commonrepo.NewRegistryNamespaceColl().FindAll(&commonrepo.FindRegOps{}) - if err == nil { - configPayload.RepoConfigs = make(map[string]*commonmodels.RegistryNamespace) - for _, repo := range repos { - // if the registry is SWR, we need to modify ak/sk according to the rule - if repo.RegProvider == config.SWRProvider { - ak := fmt.Sprintf("%s@%s", repo.Region, repo.AccessKey) - sk := util.ComputeHmacSha256(repo.AccessKey, repo.SecretKey) - repo.AccessKey = ak - repo.SecretKey = sk - } - configPayload.RepoConfigs[repo.ID.Hex()] = repo + if len(env.RegistryID) == 0 { + reg, _, err := commonservice.FindDefaultRegistry(false, log) + if err != nil { + log.Errorf("get default registry error: %v", err) + return nil, e.ErrGetCounter.AddDesc(err.Error()) } + env.RegistryID = reg.ID.Hex() + } + configPayload.RegistryID = env.RegistryID + err = SetCandidateRegistry(configPayload, log) + if err != nil { + log.Errorf("workflow_task setCandidateRegistry configPayload:[%v] err:%v", configPayload, err) + return nil, err } - configPayload.IgnoreCache = args.IgnoreCache - configPayload.ResetCache = args.ResetCache + nextTaskID, err := generateNextTaskID(args.WorkflowName) + if err != nil { + return nil, err + } + // modify configPayload + modifyConfigPayload(configPayload, args.IgnoreCache, args.ResetCache) distributeS3StoreURL, defaultS3StoreURL, err := getDefaultAndDestS3StoreURL(workflow, log) if err != nil { @@ -500,17 +605,28 @@ func CreateWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator string, } stages := make([]*commonmodels.Stage, 0) + serviceInfos := make([]*taskmodels.ServiceInfo, 0) for _, target := range args.Target { var subTasks []map[string]interface{} var err error if target.JenkinsBuildArgs == nil { - subTasks, err = BuildModuleToSubTasks("", target.Name, target.ServiceName, target.ProductName, target.Envs, env, log) + buildModuleArgs := &commonmodels.BuildModuleArgs{ + Target: target.Name, + ServiceName: target.ServiceName, + //ProductName: target.ProductName, // TODO productName may be nil in some situation, need to figure out reason + ProductName: args.ProductTmplName, + Variables: target.Envs, + Env: env, + BuildName: target.BuildName, + } + subTasks, err = BuildModuleToSubTasks(buildModuleArgs, log) } else { subTasks, err = JenkinsBuildModuleToSubTasks(&JenkinsBuildOption{ Target: target.Name, ServiceName: target.ServiceName, ProductName: args.ProductTmplName, JenkinsBuildArgs: target.JenkinsBuildArgs, + BuildName: target.BuildName, }, log) } if err != nil { @@ -519,12 +635,12 @@ func CreateWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator string, } if env != nil { - // 生成部署的subtask + // generate subtasks to deploy for _, deployEnv := range target.Deploy { if deployEnv.Type == setting.PMDeployType { continue } - deployTask, err := deployEnvToSubTasks(deployEnv, env, productTempl.Timeout) + deployTask, err := deployEnvToSubTasks(deployEnv, env, project.Timeout) if err != nil { log.Errorf("deploy env to subtask error: %v", err) return nil, e.ErrCreateTask.AddErr(err) @@ -537,6 +653,7 @@ func CreateWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator string, return nil, e.ErrCreateTask.AddErr(err) } if resetImageTask != nil { + resetImageTask["release_name"] = deployTask["release_name"] subTasks = append(subTasks, resetImageTask) } } @@ -559,8 +676,10 @@ func CreateWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator string, if distribute.Target != nil { serviceModule.ProductName = distribute.Target.ProductName } - if reflect.DeepEqual(distribute.Target, serviceModule) { + // since more field is introduced into the serviceModuleTarget structure, we only use the field we need to determine if the target match + if serviceModule.ProductName == distribute.Target.ProductName && serviceModule.ServiceName == distribute.Target.ServiceName && serviceModule.ServiceModule == distribute.Target.ServiceModule { distributeTasks, err = formatDistributeSubtasks( + serviceModule, workflow.DistributeStage.Releases, workflow.DistributeStage.ImageRepo, workflow.DistributeStage.JumpBoxHost, @@ -576,7 +695,7 @@ func CreateWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator string, subTasks = append(subTasks, distributeTasks...) } - jiraInfo, _ := poetry.GetJiraInfo(config.PoetryAPIServer(), config.PoetryAPIRootKey()) + jiraInfo, _ := systemconfig.New().GetJiraInfo() if jiraInfo != nil { jiraTask, err := AddJiraSubTask("", target.Name, target.ServiceName, args.ProductTmplName, log) if err != nil { @@ -600,7 +719,7 @@ func CreateWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator string, } // 填充subtask之间关联内容 - task := &task.Task{ + task := &taskmodels.Task{ TaskID: nextTaskID, PipelineName: args.WorkflowName, TaskCreator: taskCreator, @@ -613,7 +732,14 @@ func CreateWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator string, } sort.Sort(ByTaskKind(task.SubTasks)) - if err := ensurePipelineTask(task, args.Namespace, log); err != nil { + if err := ensurePipelineTask(&taskmodels.TaskOpt{ + Task: task, + EnvName: args.Namespace, + ServiceName: target.ServiceName, + ServiceInfos: &serviceInfos, + IsWorkflowTask: true, + ImageName: target.ImageName, + }, log); err != nil { log.Errorf("workflow_task ensurePipelineTask taskID:[%d] pipelineName:[%s] err:%v", task.ID, task.PipelineName, err) if err, ok := err.(*ContainerNotFound); ok { err := e.NewWithExtras( @@ -631,13 +757,21 @@ func CreateWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator string, } return nil, e.ErrCreateTask.AddDesc(err.Error()) } - for _, stask := range task.SubTasks { - AddSubtaskToStage(&stages, stask, target.Name) + AddSubtaskToStage(&stages, stask, target.Name+"_"+target.ServiceName) + } + } + // add extension to stage + if workflow.ExtensionStage != nil && workflow.ExtensionStage.Enabled { + extensionTask, err := addExtensionToSubTasks(workflow.ExtensionStage, serviceInfos) + if err != nil { + log.Errorf("add extension task error: %s", err) + return nil, e.ErrCreateTask.AddErr(err) } + AddSubtaskToStage(&stages, extensionTask, string(config.TaskExtension)) } - testTask := &task.Task{ + testTask := &taskmodels.Task{ TaskID: nextTaskID, PipelineName: args.WorkflowName, ProductName: args.ProductTmplName, @@ -656,12 +790,6 @@ func CreateWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator string, return nil, e.ErrCreateTask.AddDesc(err.Error()) } - err = SetCandidateRegistry(configPayload, log) - if err != nil { - log.Errorf("workflow_task setCandidateRegistry configPayload:[%v] err:%v", configPayload, err) - return nil, err - } - AddSubtaskToStage(&stages, testSubTask, testTask.TestModuleName) } @@ -669,26 +797,28 @@ func CreateWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator string, triggerBy := &commonmodels.TriggerBy{ CodehostID: args.CodehostID, RepoOwner: args.RepoOwner, + RepoNamespace: args.RepoNamespace, RepoName: args.RepoName, Source: args.Source, MergeRequestID: args.MergeRequestID, CommitID: args.CommitID, } - task := &task.Task{ - TaskID: nextTaskID, - Type: config.WorkflowType, - ProductName: workflow.ProductTmplName, - PipelineName: args.WorkflowName, - Description: args.Description, - TaskCreator: taskCreator, - ReqID: args.ReqID, - Status: config.StatusCreated, - Stages: stages, - WorkflowArgs: args, - ConfigPayload: configPayload, - StorageURI: defaultS3StoreURL, - ResetImage: workflow.ResetImage, - TriggerBy: triggerBy, + task := &taskmodels.Task{ + TaskID: nextTaskID, + Type: config.WorkflowType, + ProductName: workflow.ProductTmplName, + PipelineName: args.WorkflowName, + Description: args.Description, + TaskCreator: taskCreator, + ReqID: args.ReqID, + Status: config.StatusCreated, + Stages: stages, + WorkflowArgs: args, + ConfigPayload: configPayload, + StorageURI: defaultS3StoreURL, + ResetImage: workflow.ResetImage, + ResetImagePolicy: workflow.ResetImagePolicy, + TriggerBy: triggerBy, } if len(task.Stages) <= 0 { @@ -717,221 +847,372 @@ func CreateWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator string, } _ = scmnotify.NewService().UpdateWebhookComment(task, log) - resp := &CreateTaskResp{PipelineName: args.WorkflowName, TaskID: nextTaskID} + resp := &CreateTaskResp{ + ProjectName: args.ProductTmplName, + PipelineName: args.WorkflowName, + TaskID: nextTaskID, + } return resp, nil } -func AddDataToArgs(args *commonmodels.WorkflowTaskArgs, log *zap.SugaredLogger) error { - if args.Target == nil || len(args.Target) == 0 { - return errors.New("target cannot be empty") +func generateNextTaskID(workflowName string) (int64, error) { + nextTaskID, err := commonrepo.NewCounterColl().GetNextSeq(fmt.Sprintf(setting.WorkflowTaskFmt, workflowName)) + if err != nil { + log.Errorf("Counter.GetNextSeq error: %s", err) + return 0, e.ErrGetCounter.AddDesc(err.Error()) } + return nextTaskID, nil +} - builds, err := commonrepo.NewBuildColl().List(&commonrepo.BuildListOption{}) - if err != nil { - log.Errorf("[Build.List] error: %v", err) - return e.ErrListBuildModule.AddErr(err) +func modifyConfigPayload(configPayload *commonmodels.ConfigPayload, ignoreCache, resetCache bool) { + repos, err := commonservice.ListRegistryNamespaces("", true, log.SugaredLogger()) + if err == nil { + configPayload.RepoConfigs = make(map[string]*commonmodels.RegistryNamespace) + for _, repo := range repos { + configPayload.RepoConfigs[repo.ID.Hex()] = repo + } + } + + configPayload.IgnoreCache = ignoreCache + configPayload.ResetCache = resetCache + +} + +// add data to workflow args or create release image task +func AddDataToArgsOrCreateReleaseImageTask(args *commonmodels.WorkflowTaskArgs, log *zap.SugaredLogger) (*CreateTaskResp, error) { + if len(args.Target) == 0 && len(args.ReleaseImages) == 0 { + return nil, errors.New("target and release_images cannot be empty at the same time") } workflow, err := commonrepo.NewWorkflowColl().Find(args.WorkflowName) if err != nil { - log.Errorf("[Workflow.Find] error: %v", err) - return e.ErrFindWorkflow.AddErr(err) + log.Errorf("[Workflow.Find] error: %s", err) + return nil, e.ErrFindWorkflow.AddErr(err) } - - args.ProductTmplName = workflow.ProductTmplName - - // 补全target信息 - for _, target := range args.Target { - target.HasBuild = true - // openAPI模式，传入的name是服务名称 - // 只支持k8s服务 - opt := &commonrepo.ServiceFindOption{ - ServiceName: target.Name, - ProductName: workflow.ProductTmplName, - Type: setting.K8SDeployType, - ExcludeStatus: setting.ProductStatusDeleting} - serviceTmpl, err := commonrepo.NewServiceColl().Find(opt) + if len(args.Target) > 0 { + builds, err := commonrepo.NewBuildColl().List(&commonrepo.BuildListOption{}) if err != nil { - log.Errorf("[ServiceTmpl.Find] error: %v", err) - return e.ErrGetService.AddErr(err) + log.Errorf("[Build.List] error: %s", err) + return nil, e.ErrListBuildModule.AddErr(err) } + args.ProductTmplName = workflow.ProductTmplName + + // Complete target information + for _, target := range args.Target { + target.HasBuild = true + // OpenAPI mode, the name passed in is the service name + serviceType, err := getValidServiceType(target.ServiceType) + if err != nil { + return nil, err + } + opt := &commonrepo.ServiceFindOption{ + ServiceName: target.Name, + ProductName: workflow.ProductTmplName, + Type: serviceType, + ExcludeStatus: setting.ProductStatusDeleting} + serviceTmpl, err := commonrepo.NewServiceColl().Find(opt) + if err != nil { + log.Errorf("[ServiceTmpl.Find] error: %s", err) + return nil, e.ErrGetService.AddErr(err) + } - // 补全target中的deploy信息 - deploys := make([]commonmodels.DeployEnv, 0) - for _, container := range serviceTmpl.Containers { - // 如果服务中的服务组件存在构建，将该服务组件放进deploy + // Complete the deploy information in the target + deploys := make([]commonmodels.DeployEnv, 0) + for _, container := range serviceTmpl.Containers { + // If the service component in the service is built, put the service component into deploy + for _, build := range builds { + for _, moduleTarget := range build.Targets { + serviceModuleTarget := &commonmodels.ServiceModuleTarget{ + ProductName: serviceTmpl.ProductName, + ServiceName: serviceTmpl.ServiceName, + ServiceModule: container.Name, + } + if reflect.DeepEqual(moduleTarget, serviceModuleTarget) { + deployEnv := commonmodels.DeployEnv{Env: serviceTmpl.ServiceName + "/" + container.Name, Type: serviceTmpl.Type, ProductName: serviceTmpl.ProductName} + deploys = append(deploys, deployEnv) + } + } + } + } + target.Deploy = deploys + + // Complete the build information in the target for _, build := range builds { - for _, moduleTarget := range build.Targets { + if len(build.Targets) == 0 { + continue + } + // If the service component owned by the build contains any service component of the service, the match is considered successful + match := false + repos := build.Repos + for _, container := range serviceTmpl.Containers { serviceModuleTarget := &commonmodels.ServiceModuleTarget{ ProductName: serviceTmpl.ProductName, ServiceName: serviceTmpl.ServiceName, ServiceModule: container.Name, } - if reflect.DeepEqual(moduleTarget, serviceModuleTarget) { - deployEnv := commonmodels.DeployEnv{Env: serviceTmpl.ServiceName + "/" + container.Name, Type: serviceTmpl.Type, ProductName: serviceTmpl.ProductName} - deploys = append(deploys, deployEnv) + for _, buildTarget := range build.Targets { + if reflect.DeepEqual(buildTarget, serviceModuleTarget) { + target.Name = container.Name + target.ServiceName = serviceTmpl.ServiceName + match = true + repos = buildTarget.Repos + break + } } } - } - } - target.Deploy = deploys + if !match { + continue + } + // If the service component is successfully matched, match the warehouse name from the built warehouse list and add the repo information + if target.Build == nil { + continue + } - // 补全target中的build信息 - for _, build := range builds { - if len(build.Targets) == 0 { - continue - } - // 如果该构建拥有的服务组件 包含 该服务的任一服务组件，则认为匹配成功 - match := false - for _, container := range serviceTmpl.Containers { - serviceModuleTarget := &commonmodels.ServiceModuleTarget{ - ProductName: serviceTmpl.ProductName, - ServiceName: serviceTmpl.ServiceName, - ServiceModule: container.Name, - } - for _, buildTarget := range build.Targets { - if reflect.DeepEqual(buildTarget, serviceModuleTarget) { - target.Name = container.Name - match = true - break + for _, buildRepo := range repos { + for _, targetRepo := range target.Build.Repos { + if targetRepo.RepoName == buildRepo.RepoName { + // openAPI only pass repoName, branch, pr + targetRepo.Source = buildRepo.Source + targetRepo.RepoOwner = buildRepo.RepoOwner + targetRepo.RepoNamespace = buildRepo.RepoNamespace + targetRepo.RemoteName = buildRepo.RemoteName + targetRepo.CodehostID = buildRepo.CodehostID + targetRepo.CheckoutPath = buildRepo.CheckoutPath + } } } } - if !match { - continue - } - // 服务组件匹配成功，则从该构建的仓库列表 匹配仓库名称，并添加repo信息 - if target.Build == nil { - continue + } + // Complete test information + if workflow.TestStage != nil && workflow.TestStage.Enabled { + tests := make([]*commonmodels.TestArgs, 0) + for _, testName := range workflow.TestStage.TestNames { + moduleTest, err := commonrepo.NewTestingColl().Find(testName, "") + if err != nil { + log.Errorf("[Testing.Find] TestModuleName:%s, error:%s", testName, err) + continue + } + test := &commonmodels.TestArgs{ + TestModuleName: testName, + Namespace: args.Namespace, + Builds: moduleTest.Repos, + } + tests = append(tests, test) } - for _, buildRepo := range build.Repos { - for _, targetRepo := range target.Build.Repos { - if targetRepo.RepoName == buildRepo.RepoName { - // openAPI仅上传了repoName、branch、pr - targetRepo.Source = buildRepo.Source - targetRepo.RepoOwner = buildRepo.RepoOwner - targetRepo.RemoteName = buildRepo.RemoteName - targetRepo.CodehostID = buildRepo.CodehostID - targetRepo.CheckoutPath = buildRepo.CheckoutPath - } + + for _, testEntity := range workflow.TestStage.Tests { + moduleTest, err := commonrepo.NewTestingColl().Find(testEntity.Name, "") + if err != nil { + log.Errorf("[Testing.Find] TestModuleName:%s, error:%s", testEntity.Name, err) + continue } + test := &commonmodels.TestArgs{ + TestModuleName: testEntity.Name, + Namespace: args.Namespace, + Builds: moduleTest.Repos, + } + tests = append(tests, test) } + args.Tests = tests } + + return nil, nil } - // 补全test信息 - if workflow.TestStage != nil && workflow.TestStage.Enabled { - tests := make([]*commonmodels.TestArgs, 0) - for _, testName := range workflow.TestStage.TestNames { - moduleTest, err := commonrepo.NewTestingColl().Find(testName, "") - if err != nil { - log.Errorf("[Testing.Find] TestModuleName:%s, error:%v", testName, err) - continue - } - test := &commonmodels.TestArgs{ - TestModuleName: testName, - Namespace: args.Namespace, - Builds: moduleTest.Repos, - } - tests = append(tests, test) + + return createReleaseImageTask(workflow, args, log) +} + +func buildRegistryMap() (map[string]*commonmodels.RegistryNamespace, error) { + registries, err := commonservice.ListRegistryNamespaces("", true, log.SugaredLogger()) + if err != nil { + return nil, fmt.Errorf("failed to query registries") + } + ret := make(map[string]*commonmodels.RegistryNamespace) + for _, singleRegistry := range registries { + fullUrl := fmt.Sprintf("%s/%s", singleRegistry.RegAddr, singleRegistry.Namespace) + fullUrl = strings.TrimSuffix(fullUrl, "/") + u, _ := url.Parse(fullUrl) + if len(u.Scheme) > 0 { + fullUrl = strings.TrimPrefix(fullUrl, fmt.Sprintf("%s://", u.Scheme)) } + ret[fullUrl] = singleRegistry + } + return ret, nil +} + +func createReleaseImageTask(workflow *commonmodels.Workflow, args *commonmodels.WorkflowTaskArgs, log *zap.SugaredLogger) (*CreateTaskResp, error) { + // Get global configPayload + configPayload := commonservice.GetConfigPayload(0) + args.ProductTmplName = workflow.ProductTmplName + nextTaskID, err := generateNextTaskID(workflow.Name) + if err != nil { + return nil, err + } + // modify configPayload + modifyConfigPayload(configPayload, false, false) - for _, testEntity := range workflow.TestStage.Tests { - moduleTest, err := commonrepo.NewTestingColl().Find(testEntity.Name, "") + registryMap, err := buildRegistryMap() + if err != nil { + log.Errorf("failed to build registry map, err: %s", err) + // use default registry + reg, _, err := commonservice.FindDefaultRegistry(true, log) + if err != nil { + log.Errorf("can't find default candidate registry, err: %s", err) + return nil, e.ErrFindRegistry.AddDesc(err.Error()) + } + configPayload.Registry.Addr = reg.RegAddr + configPayload.Registry.AccessKey = reg.AccessKey + configPayload.Registry.SecretKey = reg.SecretKey + configPayload.Registry.Namespace = reg.Namespace + } else { + // extract registry from image + for _, releaseImage := range args.ReleaseImages { + registryUrl, err := commonservice.ExtractImageRegistry(releaseImage.Image) if err != nil { - log.Errorf("[Testing.Find] TestModuleName:%s, error:%v", testEntity.Name, err) + log.Errorf("failed to extract image registry, image:%s err: %s", releaseImage.Image, err) continue } - test := &commonmodels.TestArgs{ - TestModuleName: testEntity.Name, - Namespace: args.Namespace, - Builds: moduleTest.Repos, + registryUrl = strings.TrimSuffix(registryUrl, "/") + if reg, ok := registryMap[registryUrl]; ok { + configPayload.Registry.Addr = reg.RegAddr + configPayload.Registry.AccessKey = reg.AccessKey + configPayload.Registry.SecretKey = reg.SecretKey + configPayload.Registry.Namespace = reg.Namespace + break } - tests = append(tests, test) } - args.Tests = tests } - return nil -} - -func HasPermission(productName, envName string, args *commonmodels.WorkflowTaskArgs, userID int, superUser bool, log *zap.SugaredLogger) bool { - //排除触发器触发和工作流没有绑定环境的情况 - if userID == permission.AnonymousUserID || envName == "" { - return true - } - // 只有测试任务的情况 - if len(args.Target) == 0 && len(args.Tests) > 0 { - return true - } - //权限判断 - if superUser { - return true - } - poetryClient := poetry.New(config.PoetryAPIServer(), config.PoetryAPIRootKey()) - productNameMap, err := poetryClient.GetUserProject(userID, log) + distributeS3StoreURL, defaultS3StoreURL, err := getDefaultAndDestS3StoreURL(workflow, log) if err != nil { - log.Errorf("Collection.Product.List GetUserProject error: %v", err) - return false - } - //判断环境是类生产环境还是测试环境 - isProd := false - opt := &commonrepo.ProductFindOptions{Name: productName, EnvName: envName} - prod, _ := commonrepo.NewProductColl().Find(opt) - if prod != nil && prod.ClusterID != "" { - if cluster, _ := commonrepo.NewK8SClusterColl().Get(prod.ClusterID); cluster != nil { - isProd = cluster.Production - } + log.Errorf("getDefaultAndDestS3StoreUrl workflow name:[%s] err:%s", workflow.Name, err) + return nil, e.ErrCreateTask.AddErr(err) } - for _, roleID := range productNameMap[productName] { - if roleID == poetry.ProjectOwner { - return true - } - if envRolePermissions, _ := poetryClient.ListEnvRolePermission(productName, envName, roleID, log); len(envRolePermissions) > 0 { - for _, envRolePermission := range envRolePermissions { - if envRolePermission.PermissionUUID == permission.TestEnvManageUUID { - return true - } - } - } else { - if !isProd { - if poetryClient.HasOperatePermission(productName, permission.TestUpdateEnvUUID, userID, superUser, log) { - return true + + stages := make([]*commonmodels.Stage, 0) + // Generate distributed subtask + if workflow.DistributeStage != nil && workflow.DistributeStage.Enabled { + var ( + distributeTasks []map[string]interface{} + err error + subTasks = make([]map[string]interface{}, 0) + ) + + for _, imageInfo := range args.ReleaseImages { + for _, distribute := range workflow.DistributeStage.Distributes { + if distribute.Target == nil { + continue } - if poetryClient.HasOperatePermission(productName, permission.TestEnvManageUUID, userID, superUser, log) { - return true + if distribute.Target.ServiceModule == imageInfo.ServiceModule && distribute.Target.ServiceName == imageInfo.ServiceName { + distributeTasks, err = formatDistributeSubtasks( + &commonmodels.ServiceModuleTarget{ + ProductName: args.ProductTmplName, + ServiceName: imageInfo.ServiceName, + ServiceModule: imageInfo.ServiceModule, + }, + workflow.DistributeStage.Releases, + workflow.DistributeStage.ImageRepo, + workflow.DistributeStage.JumpBoxHost, + distributeS3StoreURL, + distribute, + ) + if err != nil { + log.Errorf("distrbiute stages to subtasks error: %s", err) + return nil, e.ErrCreateTask.AddErr(err) + } } + } + subTasks = append(subTasks, distributeTasks...) - envRolePermissions, _ := poetryClient.ListEnvRolePermission(productName, envName, 0, log) - for _, envRolePermission := range envRolePermissions { - if envRolePermission.PermissionUUID == permission.TestEnvManageUUID { - return true - } + // Fill in the associated content between subtasks + task := &taskmodels.Task{ + TaskID: nextTaskID, + PipelineName: workflow.Name, + TaskCreator: setting.RequestModeOpenAPI, + ReqID: args.ReqID, + SubTasks: subTasks, + ServiceName: imageInfo.ServiceModule, + ConfigPayload: configPayload, + TaskArgs: &commonmodels.TaskArgs{PipelineName: workflow.Name, TaskCreator: setting.RequestModeOpenAPI, Deploy: commonmodels.DeployArgs{Image: imageInfo.Image}}, + ProductName: workflow.ProductTmplName, + } + sort.Sort(ByTaskKind(task.SubTasks)) + + if err := ensurePipelineTask(&taskmodels.TaskOpt{ + Task: task, + }, log); err != nil { + log.Errorf("workflow_task ensurePipelineTask taskID:[%d] pipelineName:[%s] err:%s", task.ID, task.PipelineName, err) + if err, ok := err.(*ContainerNotFound); ok { + err := e.NewWithExtras( + e.ErrCreateTaskFailed.AddErr(err), + "container doesn't exists", map[string]interface{}{ + "productName": err.ProductName, + "envName": err.EnvName, + "serviceName": err.ServiceName, + "containerName": err.Container, + }) + return nil, err } - } else { - if poetryClient.HasOperatePermission(productName, permission.ProdEnvManageUUID, userID, superUser, log) { - return true + if _, ok := err.(*ImageIllegal); ok { + return nil, e.ErrCreateTask.AddDesc("IMAGE is illegal") } + return nil, e.ErrCreateTask.AddDesc(err.Error()) + } - envRolePermissions, _ := poetryClient.ListEnvRolePermission(productName, envName, 0, log) - for _, envRolePermission := range envRolePermissions { - if envRolePermission.PermissionUUID == permission.ProdEnvManageUUID { - return true - } - } + for _, stask := range task.SubTasks { + AddSubtaskToStage(&stages, stask, imageInfo.ServiceModule) } } } - // 如果该项目设置过all-users,判断all-users的权限 - productRole, _ := poetryClient.ListRoles(productName, log) - if productRole != nil { - permissionUUIDs, _ := poetryClient.GetUserPermissionUUIDs(productRole.ID, productName, log) - if isProd && sets.NewString(permissionUUIDs...).Has(permission.ProdEnvManageUUID) { - return true - } else if !isProd && sets.NewString(permissionUUIDs...).Has(permission.TestEnvManageUUID) { - return true - } + + sort.Sort(ByStageKind(stages)) + task := &taskmodels.Task{ + WorkflowArgs: args, + TaskID: nextTaskID, + Type: config.WorkflowType, + ProductName: workflow.ProductTmplName, + PipelineName: workflow.Name, + TaskCreator: setting.RequestModeOpenAPI, + ReqID: args.ReqID, + Status: config.StatusCreated, + Stages: stages, + ConfigPayload: configPayload, + StorageURI: defaultS3StoreURL, + } + + if len(task.Stages) <= 0 { + errMessage := fmt.Sprintf("%s or %s", e.PipelineSubTaskNotFoundErrMsg, "Invalid service module") + return nil, e.ErrCreateTask.AddDesc(errMessage) + } + + endpoint := fmt.Sprintf("%s-%s:9000", config.Namespace(), ClusterStorageEP) + task.StorageEndpoint = endpoint + + if err := CreateTask(task); err != nil { + log.Errorf("workflow Create task:[%v] err:%s", task, err) + return nil, e.ErrCreateTask + } + + resp := &CreateTaskResp{ + ProjectName: workflow.ProductTmplName, + PipelineName: workflow.Name, + TaskID: nextTaskID, + } + + return resp, nil +} + +// Only supports k8s and helm two service types currently +func getValidServiceType(serviceType string) (string, error) { + switch serviceType { + // Compatible when the service_type is equal to empty + case setting.K8SDeployType, "": + return setting.K8SDeployType, nil + case setting.HelmDeployType: + return setting.HelmDeployType, nil + default: + return "", fmt.Errorf("Unsupported service type") } - return false } func dealWithNamespace(args *commonmodels.WorkflowTaskArgs) { @@ -1047,7 +1328,7 @@ func GetS3Storage(id string, logger *zap.SugaredLogger) (*commonmodels.S3Storage func deployEnvToSubTasks(env commonmodels.DeployEnv, prodEnv *commonmodels.Product, timeout int) (map[string]interface{}, error) { var ( resp map[string]interface{} - deployTask = task.Deploy{ + deployTask = taskmodels.Deploy{ TaskType: config.TaskDeploy, Enabled: true, Namespace: prodEnv.Namespace, @@ -1065,7 +1346,7 @@ func deployEnvToSubTasks(env commonmodels.DeployEnv, prodEnv *commonmodels.Produ return nil, err } deployTask.ServiceName = envList[0] - deployTask.ContainerName = envList[1] + deployTask.ContainerName = envList[1] + "_" + envList[0] switch env.Type { case setting.K8SDeployType: @@ -1073,14 +1354,18 @@ func deployEnvToSubTasks(env commonmodels.DeployEnv, prodEnv *commonmodels.Produ return deployTask.ToSubTask() case setting.HelmDeployType: deployTask.ServiceType = setting.HelmDeployType - for _, services := range prodEnv.Services { - for _, service := range services { - if service.ServiceName == deployTask.ServiceName { - deployTask.ServiceRevision = service.Revision - return deployTask.ToSubTask() - } - } + if pSvc, ok := prodEnv.GetServiceMap()[deployTask.ServiceName]; ok { + deployTask.ServiceRevision = pSvc.Revision + } + revisionSvc, err := commonrepo.NewServiceColl().Find(&commonrepo.ServiceFindOption{ + ServiceName: deployTask.ServiceName, + Revision: deployTask.ServiceRevision, + ProductName: prodEnv.ProductName, + }) + if err != nil { + return nil, fmt.Errorf("failed to find service: %s with revision: %d, err: %s", deployTask.ServiceName, deployTask.ServiceRevision, err) } + deployTask.ReleaseName = util.GeneReleaseName(revisionSvc.GetReleaseNaming(), prodEnv.ProductName, prodEnv.Namespace, prodEnv.EnvName, deployTask.ServiceName) return deployTask.ToSubTask() } return resp, fmt.Errorf("env type not match") @@ -1089,7 +1374,22 @@ func deployEnvToSubTasks(env commonmodels.DeployEnv, prodEnv *commonmodels.Produ func resetImageTaskToSubTask(env commonmodels.DeployEnv, prodEnv *commonmodels.Product) (map[string]interface{}, error) { switch env.Type { case setting.K8SDeployType: - deployTask := task.Deploy{TaskType: config.TaskResetImage, Enabled: true} + deployTask := taskmodels.Deploy{TaskType: config.TaskResetImage, Enabled: true} + deployTask.Namespace = prodEnv.Namespace + deployTask.ProductName = prodEnv.ProductName + deployTask.SkipWaiting = true + deployTask.EnvName = prodEnv.EnvName + envList := strings.Split(env.Env, "/") + if len(envList) != 2 { + err := fmt.Errorf("[%s]split target env error", env.Env) + log.Error(err) + return nil, err + } + deployTask.ServiceName = envList[0] + deployTask.ContainerName = envList[1] + return deployTask.ToSubTask() + case setting.HelmDeployType: + deployTask := taskmodels.Deploy{TaskType: config.TaskResetImage, Enabled: true} deployTask.Namespace = prodEnv.Namespace deployTask.ProductName = prodEnv.ProductName deployTask.SkipWaiting = true @@ -1102,6 +1402,7 @@ func resetImageTaskToSubTask(env commonmodels.DeployEnv, prodEnv *commonmodels.P } deployTask.ServiceName = envList[0] deployTask.ContainerName = envList[1] + deployTask.ServiceType = setting.HelmDeployType return deployTask.ToSubTask() default: return nil, nil @@ -1109,23 +1410,70 @@ func resetImageTaskToSubTask(env commonmodels.DeployEnv, prodEnv *commonmodels.P } func artifactToSubTasks(name, image string) (map[string]interface{}, error) { - artifactTask := task.Artifact{TaskType: config.TaskArtifact, Enabled: true} + artifactTask := taskmodels.Artifact{TaskType: config.TaskArtifact, Enabled: true} artifactTask.Name = name artifactTask.Image = image - return artifactTask.ToSubTask() } -func formatDistributeSubtasks(releaseImages []commonmodels.RepoImage, imageRepo, jumpboxHost, destStorageURL string, distribute *commonmodels.ProductDistribute) ([]map[string]interface{}, error) { +func formatDistributeSubtasks(serviceModule *commonmodels.ServiceModuleTarget, releaseImages []commonmodels.RepoImage, imageRepo, jumpboxHost, destStorageURL string, distribute *commonmodels.ProductDistribute) ([]map[string]interface{}, error) { var resp []map[string]interface{} - + productName := serviceModule.ProductName if distribute.ImageDistribute { - t := task.ReleaseImage{ - TaskType: config.TaskReleaseImage, - Enabled: true, - ImageRepo: imageRepo, - Releases: releaseImages, + t := taskmodels.ReleaseImage{ + TaskType: config.TaskReleaseImage, + Enabled: true, + ProductName: productName, } + // get product Info for further use + productInfo, err := template.NewProductColl().Find(productName) + if err != nil { + return nil, err + } + distributeInfo := make([]*taskmodels.DistributeInfo, 0) + // now we add distributeInfo in + for _, repoInfo := range releaseImages { + envInfo, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + EnvName: repoInfo.DeployEnv, + Name: productName, + }) + if err != nil { + return nil, err + } + releaseName := "" + if repoInfo.DeployEnabled { + if productInfo.ProductFeature != nil && productInfo.ProductFeature.DeployType == setting.HelmDeployType { + svcMap := envInfo.GetServiceMap() + pSvc, ok := svcMap[serviceModule.ServiceName] + if !ok { + return nil, fmt.Errorf("can't find service: %s in product: %s:%s", serviceModule.ServiceName, productName, repoInfo.DeployEnv) + } + templateSvc, err := commonrepo.NewServiceColl().Find(&commonrepo.ServiceFindOption{ + ServiceName: serviceModule.ServiceName, + Revision: pSvc.Revision, + Type: setting.HelmDeployType, + ProductName: productName, + }) + if err != nil { + return nil, err + } + releaseName = util.GeneReleaseName(templateSvc.GetReleaseNaming(), productName, envInfo.Namespace, repoInfo.DeployEnv, serviceModule.ServiceName) + } + } + distributeInfo = append(distributeInfo, &taskmodels.DistributeInfo{ + DeployEnabled: repoInfo.DeployEnabled, + DeployEnv: repoInfo.DeployEnv, + DeployServiceType: productInfo.ProductFeature.DeployType, + DeployClusterID: envInfo.ClusterID, + DeployNamespace: envInfo.Namespace, + ReleaseName: releaseName, + RepoID: repoInfo.RepoID, + }) + } + t.DistributeInfo = distributeInfo + t.Releases = releaseImages + + // convert to subtask subtask, err := t.ToSubTask() if err != nil { return resp, err @@ -1133,7 +1481,7 @@ func formatDistributeSubtasks(releaseImages []commonmodels.RepoImage, imageRepo, resp = append(resp, subtask) } if distribute.QstackDistribute && destStorageURL != "" { - task := task.DistributeToS3{ + task := taskmodels.DistributeToS3{ TaskType: config.TaskDistributeToS3, Enabled: true, DestStorageURL: destStorageURL, @@ -1165,7 +1513,7 @@ func AddJiraSubTask(moduleName, target, serviceName, productName string, log *za if err != nil { return nil, e.ErrConvertSubTasks.AddErr(err) } - jira := &task.Jira{ + jira := &taskmodels.Jira{ TaskType: config.TaskJira, Enabled: true, } @@ -1177,12 +1525,26 @@ func AddJiraSubTask(moduleName, target, serviceName, productName string, log *za } func addSecurityToSubTasks() (map[string]interface{}, error) { - securityTask := task.Security{TaskType: config.TaskSecurity, Enabled: true} + securityTask := taskmodels.Security{TaskType: config.TaskSecurity, Enabled: true} return securityTask.ToSubTask() } +func addExtensionToSubTasks(stage *commonmodels.ExtensionStage, serviceInfos []*taskmodels.ServiceInfo) (map[string]interface{}, error) { + extensionTask := taskmodels.Extension{ + TaskType: config.TaskExtension, + Enabled: true, + URL: stage.URL, + Path: stage.Path, + Headers: stage.Headers, + IsCallback: stage.IsCallback, + Timeout: stage.Timeout, + ServiceInfos: serviceInfos, + } + return extensionTask.ToSubTask() +} + func workFlowArgsToTaskArgs(target string, workflowArgs *commonmodels.WorkflowTaskArgs) *commonmodels.TaskArgs { - resp := &commonmodels.TaskArgs{PipelineName: workflowArgs.WorkflowName, TaskCreator: workflowArgs.WorklowTaskCreator} + resp := &commonmodels.TaskArgs{PipelineName: workflowArgs.WorkflowName, TaskCreator: workflowArgs.WorkflowTaskCreator} for _, build := range workflowArgs.Target { if build.Name == target { if build.Build != nil { @@ -1194,8 +1556,10 @@ func workFlowArgsToTaskArgs(target string, workflowArgs *commonmodels.WorkflowTa } // TODO 和validation中转化testsubtask合并为一个方法 -func testArgsToSubtask(args *commonmodels.WorkflowTaskArgs, pt *task.Task, log *zap.SugaredLogger) ([]*task.Testing, error) { - var resp []*task.Testing +func testArgsToSubtask(args *commonmodels.WorkflowTaskArgs, pt *taskmodels.Task, log *zap.SugaredLogger) ([]*taskmodels.Testing, error) { + var resp []*taskmodels.Testing + var servicesArray []string + var services string // 创建任务的测试参数为脱敏数据，需要转换为实际数据 for _, test := range args.Tests { @@ -1205,10 +1569,15 @@ func testArgsToSubtask(args *commonmodels.WorkflowTaskArgs, pt *task.Task, log * } } + for _, service := range args.Target { + servicesArray = append(servicesArray, service.ServiceName) + } + services = strings.Join(servicesArray, ",") + testArgs := args.Tests - testCreator := args.WorklowTaskCreator + testCreator := args.WorkflowTaskCreator - registries, err := commonservice.ListRegistryNamespaces(log) + registries, err := commonservice.ListRegistryNamespaces("", true, log) if err != nil { log.Errorf("ListRegistryNamespaces err:%v", err) } @@ -1223,8 +1592,17 @@ func testArgsToSubtask(args *commonmodels.WorkflowTaskArgs, pt *task.Task, log * log.Errorf("[%s]get TestingModule error: %v", testArg.TestModuleName, err) return resp, err } + for _, repo := range testModule.Repos { + repoInfo, err := systemconfig.New().GetCodeHost(repo.CodehostID) + if err != nil { + log.Errorf("Failed to get proxy settings for codehost ID: %d, the error is: %s", repo.CodehostID, err) + return nil, err + } + repo.EnableProxy = repoInfo.EnableProxy + repo.RepoNamespace = repo.GetRepoNamespace() + } - testTask := &task.Testing{ + testTask := &taskmodels.Testing{ TaskType: config.TaskTestingV2, Enabled: true, TestName: "test", @@ -1233,7 +1611,7 @@ func testArgsToSubtask(args *commonmodels.WorkflowTaskArgs, pt *task.Task, log * testTask.TestModuleName = testModule.Name testTask.JobCtx.TestType = testModule.TestType testTask.JobCtx.Builds = testModule.Repos - testTask.JobCtx.BuildSteps = append(testTask.JobCtx.BuildSteps, &task.BuildStep{BuildType: "shell", Scripts: testModule.Scripts}) + testTask.JobCtx.BuildSteps = append(testTask.JobCtx.BuildSteps, &taskmodels.BuildStep{BuildType: "shell", Scripts: testModule.Scripts}) testTask.JobCtx.ArtifactPaths = testModule.ArtifactPaths testTask.JobCtx.TestThreshold = testModule.Threshold @@ -1241,6 +1619,21 @@ func testArgsToSubtask(args *commonmodels.WorkflowTaskArgs, pt *task.Task, log * testTask.JobCtx.TestResultPath = testModule.TestResultPath testTask.JobCtx.TestReportPath = testModule.TestReportPath + clusterInfo, err := commonrepo.NewK8SClusterColl().Get(testModule.PreTest.ClusterID) + if err != nil { + return nil, err + } + testTask.Cache = clusterInfo.Cache + + // If the cluster is not configured with a cache medium, the cache cannot be used, so don't enable cache explicitly. + if testTask.Cache.MediumType == "" { + testTask.CacheEnable = false + } else { + testTask.CacheEnable = testModule.CacheEnable + testTask.CacheDirType = testModule.CacheDirType + testTask.CacheUserDir = testModule.CacheUserDir + } + if testTask.Registries == nil { testTask.Registries = registries } @@ -1249,6 +1642,8 @@ func testArgsToSubtask(args *commonmodels.WorkflowTaskArgs, pt *task.Task, log * testTask.InstallItems = testModule.PreTest.Installs testTask.JobCtx.CleanWorkspace = testModule.PreTest.CleanWorkspace testTask.JobCtx.EnableProxy = testModule.PreTest.EnableProxy + testTask.Namespace = testModule.PreTest.Namespace + testTask.ClusterID = testModule.PreTest.ClusterID envs := testModule.PreTest.Envs[:] @@ -1262,10 +1657,15 @@ func testArgsToSubtask(args *commonmodels.WorkflowTaskArgs, pt *task.Task, log * } } envs = append(envs, &commonmodels.KeyVal{Key: "TEST_URL", Value: GetLink(pt, configbase.SystemAddress(), config.WorkflowType)}) + envs = append(envs, &commonmodels.KeyVal{Key: "SERVICES", Value: services}) + envs = append(envs, &commonmodels.KeyVal{Key: "WORKSPACE", Value: "/workspace"}) + testTask.JobCtx.EnvVars = envs testTask.ImageID = testModule.PreTest.ImageID testTask.BuildOS = testModule.PreTest.BuildOS testTask.ImageFrom = testModule.PreTest.ImageFrom + testTask.ClusterID = testModule.PreTest.ClusterID + testTask.Namespace = testModule.PreTest.Namespace // 自定义基础镜像的镜像名称可能会被更新，需要使用ID获取最新的镜像名称 if testModule.PreTest.ImageID != "" { basicImage, err := commonrepo.NewBasicImageColl().Find(testModule.PreTest.ImageID) @@ -1276,6 +1676,7 @@ func testArgsToSubtask(args *commonmodels.WorkflowTaskArgs, pt *task.Task, log * } } testTask.ResReq = testModule.PreTest.ResReq + testTask.ResReqSpec = testModule.PreTest.ResReqSpec } // 设置 build 安装脚本 testTask.InstallCtx, err = buildInstallCtx(testTask.InstallItems) @@ -1284,7 +1685,7 @@ func testArgsToSubtask(args *commonmodels.WorkflowTaskArgs, pt *task.Task, log * return resp, err } - // Iterate test jobctx builds, and replace it if params specified from task. + // Iterate test jobctx builds, and replace it if params specified from taskmodels. // 外部触发的pipeline if testCreator == setting.WebhookTaskCreator || testCreator == setting.CronTaskCreator { _ = SetTriggerBuilds(testTask.JobCtx.Builds, testArg.Builds, log) @@ -1298,7 +1699,7 @@ func testArgsToSubtask(args *commonmodels.WorkflowTaskArgs, pt *task.Task, log * return resp, nil } -func CreateArtifactWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator string, userID int, superUser bool, log *zap.SugaredLogger) (*CreateTaskResp, error) { +func CreateArtifactWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator string, log *zap.SugaredLogger) (*CreateTaskResp, error) { if args == nil { return nil, fmt.Errorf("args should not be nil") } @@ -1314,11 +1715,6 @@ func CreateArtifactWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator return nil, e.ErrFindWorkflow.AddDesc(err.Error()) } - if !HasPermission(workflow.ProductTmplName, workflow.EnvName, args, userID, superUser, log) { - log.Warnf("该工作流[%s]绑定的环境您没有权限,用户[%s]不能执行该工作流!", workflow.Name, taskCreator) - return nil, e.ErrCreateTask.AddDesc("该工作流绑定的环境您没有更新环境或者环境管理权限,不能执行该工作流") - } - var env *commonmodels.Product if args.Namespace != "" { // 查找要部署的环境 @@ -1343,7 +1739,7 @@ func CreateArtifactWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator // 获取全局configpayload configPayload := commonservice.GetConfigPayload(args.CodehostID) - repos, err := commonrepo.NewRegistryNamespaceColl().FindAll(&commonrepo.FindRegOps{}) + repos, err := commonservice.ListRegistryNamespaces("", true, log) if err == nil { configPayload.RepoConfigs = make(map[string]*commonmodels.RegistryNamespace) for _, repo := range repos { @@ -1351,6 +1747,12 @@ func CreateArtifactWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator } } + err = SetCandidateRegistry(configPayload, log) + if err != nil { + log.Errorf("workflow_task setCandidateRegistry configPayload:[%v] err:%v", configPayload, err) + return nil, err + } + configPayload.IgnoreCache = args.IgnoreCache configPayload.ResetCache = args.ResetCache @@ -1363,34 +1765,55 @@ func CreateArtifactWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator stages := make([]*commonmodels.Stage, 0) for _, artifact := range args.Artifact { subTasks := make([]map[string]interface{}, 0) - artifactSubtask, err := artifactToSubTasks(artifact.Name, artifact.Image) - if err != nil { - log.Errorf("artifactToSubTasks artifact.Name:[%s] err:%v", artifact.Name, err) - return nil, e.ErrCreateTask.AddErr(err) - } - subTasks = append(subTasks, artifactSubtask) - if env != nil { - // 生成部署的subtask - for _, deployEnv := range artifact.Deploy { - deployTask, err := deployEnvToSubTasks(deployEnv, env, productTempl.Timeout) - if err != nil { - log.Errorf("deploy env to subtask error: %v", err) - return nil, err - } - - if workflow.ResetImage { - resetImageTask, err := resetImageTaskToSubTask(deployEnv, env) + // image artifact deploy + if artifact.Image != "" { + artifactSubtask, err := artifactToSubTasks(artifact.Name, artifact.Image) + if err != nil { + log.Errorf("artifactToSubTasks artifact.Name:[%s] err:%v", artifact.Name, err) + return nil, e.ErrCreateTask.AddErr(err) + } + subTasks = append(subTasks, artifactSubtask) + if env != nil { + // 生成部署的subtask + for _, deployEnv := range artifact.Deploy { + deployTask, err := deployEnvToSubTasks(deployEnv, env, productTempl.Timeout) if err != nil { - log.Errorf("resetImageTaskToSubTask deploy env:[%s] err:%v ", deployEnv.Env, err) + log.Errorf("deploy env to subtask error: %v", err) return nil, err } - if resetImageTask != nil { - subTasks = append(subTasks, resetImageTask) - } - } - subTasks = append(subTasks, deployTask) + if workflow.ResetImage { + resetImageTask, err := resetImageTaskToSubTask(deployEnv, env) + if err != nil { + log.Errorf("resetImageTaskToSubTask deploy env:[%s] err:%v ", deployEnv.Env, err) + return nil, err + } + if resetImageTask != nil { + resetImageTask["release_name"] = deployTask["release_name"] + subTasks = append(subTasks, resetImageTask) + } + } + subTasks = append(subTasks, deployTask) + } + } + } else if artifact.FileName != "" { + buildModuleArgs := &commonmodels.BuildModuleArgs{ + Target: artifact.Name, + ServiceName: artifact.ServiceName, + ProductName: args.ProductTmplName, + Env: env, + URL: artifact.URL, + WorkflowName: artifact.WorkflowName, + TaskID: artifact.TaskID, + FileName: artifact.FileName, + TaskType: string(config.TaskArtifactDeploy), + } + buildSubtasks, err := BuildModuleToSubTasks(buildModuleArgs, log) + if err != nil { + log.Errorf("buildModuleToSubTasks target:[%s] err:%s", artifact.Name, err) + return nil, e.ErrCreateTask.AddErr(err) } + subTasks = append(subTasks, buildSubtasks...) } // 生成分发的subtask @@ -1409,6 +1832,7 @@ func CreateArtifactWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator } if reflect.DeepEqual(distribute.Target, serviceModule) { distributeTasks, err = formatDistributeSubtasks( + serviceModule, workflow.DistributeStage.Releases, workflow.DistributeStage.ImageRepo, workflow.DistributeStage.JumpBoxHost, @@ -1438,18 +1862,25 @@ func CreateArtifactWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator } // 填充subtask之间关联内容 - task := &task.Task{ + task := &taskmodels.Task{ + ProductName: args.ProductTmplName, + PipelineName: args.WorkflowName, TaskID: nextTaskID, TaskCreator: taskCreator, ReqID: args.ReqID, SubTasks: subTasks, ServiceName: artifact.Name, ConfigPayload: configPayload, + TaskArgs: workFlowArgsToTaskArgs(artifact.Name, args), WorkflowArgs: args, } sort.Sort(ByTaskKind(task.SubTasks)) - if err := ensurePipelineTask(task, args.Namespace, log); err != nil { + if err := ensurePipelineTask(&taskmodels.TaskOpt{ + Task: task, + EnvName: args.Namespace, + IsWorkflowTask: true, + }, log); err != nil { log.Errorf("workflow_task ensurePipelineTask task:[%v] err:%v", task, err) if err, ok := err.(*ContainerNotFound); ok { err := e.NewWithExtras( @@ -1467,11 +1898,11 @@ func CreateArtifactWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator } for _, stask := range task.SubTasks { - AddSubtaskToStage(&stages, stask, artifact.Name) + AddSubtaskToStage(&stages, stask, artifact.Name+"_"+artifact.ServiceName) } } - testTask := &task.Task{ + testTask := &taskmodels.Task{ TaskID: nextTaskID, PipelineName: args.WorkflowName, ProductName: args.ProductTmplName, @@ -1489,13 +1920,6 @@ func CreateArtifactWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator log.Errorf("workflow_task ToSubTask err:%v", err) return nil, e.ErrCreateTask.AddDesc(err.Error()) } - - err = SetCandidateRegistry(configPayload, log) - if err != nil { - log.Errorf("workflow_task setCandidateRegistry configPayload:[%v] err:%v", configPayload, err) - return nil, err - } - AddSubtaskToStage(&stages, testSubTask, testTask.TestModuleName) } @@ -1505,21 +1929,22 @@ func CreateArtifactWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator MergeRequestID: args.MergeRequestID, CommitID: args.CommitID, } - task := &task.Task{ - TaskID: nextTaskID, - Type: config.WorkflowType, - ProductName: workflow.ProductTmplName, - PipelineName: args.WorkflowName, - Description: args.Description, - TaskCreator: taskCreator, - ReqID: args.ReqID, - Status: config.StatusCreated, - Stages: stages, - WorkflowArgs: args, - ConfigPayload: configPayload, - StorageURI: defaultS3StoreURL, - ResetImage: workflow.ResetImage, - TriggerBy: triggerBy, + task := &taskmodels.Task{ + TaskID: nextTaskID, + Type: config.WorkflowType, + ProductName: workflow.ProductTmplName, + PipelineName: args.WorkflowName, + Description: args.Description, + TaskCreator: taskCreator, + ReqID: args.ReqID, + Status: config.StatusCreated, + Stages: stages, + WorkflowArgs: args, + ConfigPayload: configPayload, + StorageURI: defaultS3StoreURL, + ResetImage: workflow.ResetImage, + ResetImagePolicy: workflow.ResetImagePolicy, + TriggerBy: triggerBy, } if len(task.Stages) <= 0 { @@ -1544,29 +1969,70 @@ func CreateArtifactWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator } _ = scmnotify.NewService().UpdateWebhookComment(task, log) - resp := &CreateTaskResp{PipelineName: args.WorkflowName, TaskID: nextTaskID} + resp := &CreateTaskResp{ + ProjectName: workflow.ProductTmplName, + PipelineName: args.WorkflowName, + TaskID: nextTaskID, + } return resp, nil } -func BuildModuleToSubTasks(moduleName, target, serviceName, productName string, envs []*commonmodels.KeyVal, pro *commonmodels.Product, log *zap.SugaredLogger) ([]map[string]interface{}, error) { +// fillBuildDetail fill the contents for builds created from build templates +func fillBuildDetail(moduleBuild *commonmodels.Build, serviceName, serviceModule string) error { + if moduleBuild.TemplateID == "" { + return nil + } + buildTemplate, err := commonrepo.NewBuildTemplateColl().Find(&commonrepo.BuildTemplateQueryOption{ + ID: moduleBuild.TemplateID, + }) + if err != nil { + return fmt.Errorf("failed to find build template with id: %s, err: %s", moduleBuild.TemplateID, err) + } + + moduleBuild.Timeout = buildTemplate.Timeout + moduleBuild.PreBuild = buildTemplate.PreBuild + moduleBuild.JenkinsBuild = buildTemplate.JenkinsBuild + moduleBuild.Scripts = buildTemplate.Scripts + moduleBuild.PostBuild = buildTemplate.PostBuild + moduleBuild.SSHs = buildTemplate.SSHs + moduleBuild.PMDeployScripts = buildTemplate.PMDeployScripts + moduleBuild.CacheEnable = buildTemplate.CacheEnable + moduleBuild.CacheDirType = buildTemplate.CacheDirType + moduleBuild.CacheUserDir = buildTemplate.CacheUserDir + moduleBuild.AdvancedSettingsModified = buildTemplate.AdvancedSettingsModified + + // repos are configured by service modules + for _, serviceConfig := range moduleBuild.Targets { + if serviceConfig.ServiceName == serviceName && serviceConfig.ServiceModule == serviceModule { + moduleBuild.Repos = serviceConfig.Repos + if moduleBuild.PreBuild == nil { + moduleBuild.PreBuild = &commonmodels.PreBuild{} + } + moduleBuild.PreBuild.Envs = commonservice.MergeBuildEnvs(moduleBuild.PreBuild.Envs, serviceConfig.Envs) + break + } + } + return nil +} + +func BuildModuleToSubTasks(args *commonmodels.BuildModuleArgs, log *zap.SugaredLogger) ([]map[string]interface{}, error) { var ( subTasks = make([]map[string]interface{}, 0) serviceTmpl *commonmodels.Service ) - opt := &commonrepo.BuildListOption{ - Name: moduleName, - ServiceName: serviceName, - ProductName: productName, + Name: args.BuildName, + ServiceName: args.ServiceName, + ProductName: args.ProductName, } - if len(target) > 0 { - opt.Targets = []string{target} + if len(args.Target) > 0 { + opt.Targets = []string{args.Target} } - if pro != nil { + if args.Env != nil { serviceTmpl, _ = commonservice.GetServiceTemplate( - target, setting.PMDeployType, productName, setting.ProductStatusDeleting, 0, log, + args.Target, setting.PMDeployType, args.ProductName, setting.ProductStatusDeleting, 0, log, ) } @@ -1574,26 +2040,68 @@ func BuildModuleToSubTasks(moduleName, target, serviceName, productName string, if err != nil { return nil, e.ErrConvertSubTasks.AddErr(err) } + // The service may be a shared service + if len(modules) == 0 { + opt.ProductName = "" + modules, err = commonrepo.NewBuildColl().List(opt) + if err != nil { + return nil, e.ErrConvertSubTasks.AddErr(err) + } + } - registries, err := commonservice.ListRegistryNamespaces(log) + registries, err := commonservice.ListRegistryNamespaces("", true, log) if err != nil { return nil, e.ErrConvertSubTasks.AddErr(err) } for _, module := range modules { - build := &task.Build{ + err = fillBuildDetail(module, args.ServiceName, args.Target) + if err != nil { + return nil, e.ErrConvertSubTasks.AddErr(err) + } + build := &taskmodels.Build{ TaskType: config.TaskBuild, Enabled: true, InstallItems: module.PreBuild.Installs, - ServiceName: target, - Service: serviceName, - JobCtx: task.JobCtx{}, + ServiceName: args.Target, + Service: args.ServiceName, + JobCtx: taskmodels.JobCtx{}, ImageID: module.PreBuild.ImageID, BuildOS: module.PreBuild.BuildOS, ImageFrom: module.PreBuild.ImageFrom, ResReq: module.PreBuild.ResReq, + ResReqSpec: module.PreBuild.ResReqSpec, Timeout: module.Timeout, Registries: registries, + ProductName: args.ProductName, + Namespace: module.PreBuild.Namespace, + ClusterID: module.PreBuild.ClusterID, + } + + // In some old build configurations, the `pre_build.cluster_id` field is empty indicating that's a local cluster. + // We do a protection here to avoid query failure. + // Resaving the build configuration after v1.8.0 will automatically populate this field. + if module.PreBuild.ClusterID == "" { + module.PreBuild.ClusterID = setting.LocalClusterID + } + + clusterInfo, err := commonrepo.NewK8SClusterColl().Get(module.PreBuild.ClusterID) + if err != nil { + return nil, e.ErrConvertSubTasks.AddErr(err) + } + build.Cache = clusterInfo.Cache + + // If the cluster is not configured with a cache medium, the cache cannot be used, so don't enable cache explicitly. + if build.Cache.MediumType == "" { + build.CacheEnable = false + } else { + build.CacheEnable = module.CacheEnable + build.CacheDirType = module.CacheDirType + build.CacheUserDir = module.CacheUserDir + } + + if args.TaskType != "" { + build.TaskType = config.TaskArtifactDeploy } // 自定义基础镜像的镜像名称可能会被更新，需要使用ID获取最新的镜像名称 @@ -1610,39 +2118,59 @@ func BuildModuleToSubTasks(moduleName, target, serviceName, productName string, build.ImageFrom = commonmodels.ImageFromKoderover } + envHostInfos := make(map[string]*commonmodels.PrivateKey) if serviceTmpl != nil { - build.Namespace = pro.Namespace build.ServiceType = setting.PMDeployType envHost := make(map[string][]string) + envHostNames := make(map[string][]string) + for _, envConfig := range serviceTmpl.EnvConfigs { - privateKeys, err := commonrepo.NewPrivateKeyColl().ListHostIPByIDs(envConfig.HostIDs) + privateKeys, err := commonrepo.NewPrivateKeyColl().ListHostIPByArgs(&commonrepo.ListHostIPArgs{IDs: envConfig.HostIDs}) if err != nil { - log.Errorf("ListNameByIDs err:%s", err) + log.Errorf("ListNameByArgs ids err:%s", err) continue } - for _, privateKey := range privateKeys { - envHost[envConfig.EnvName] = append(envHost[envConfig.EnvName], privateKey.IP) + ips := sets.NewString() + names := sets.NewString() + ips, names = extractHost(privateKeys, ips, names, envHostInfos) + privateKeys, err = commonrepo.NewPrivateKeyColl().ListHostIPByArgs(&commonrepo.ListHostIPArgs{Labels: envConfig.Labels}) + if err != nil { + log.Errorf("ListNameByArgs labels err:%s", err) + continue } + ips, names = extractHost(privateKeys, ips, names, envHostInfos) + envHost[envConfig.EnvName] = ips.List() + envHostNames[envConfig.EnvName] = names.List() } build.EnvHostInfo = envHost + build.EnvHostNames = envHostNames } - if pro != nil { - build.EnvName = pro.EnvName + if args.Env != nil { + build.EnvName = args.Env.EnvName } if build.InstallItems == nil { build.InstallItems = make([]*commonmodels.Item, 0) } - build.JobCtx.Builds = module.Repos + build.JobCtx.Builds = module.SafeRepos() + for _, repo := range build.JobCtx.Builds { + repoInfo, err := systemconfig.New().GetCodeHost(repo.CodehostID) + if err != nil { + log.Errorf("Failed to get proxy settings for codehost ID: %d, the error is: %s", repo.CodehostID, err) + return nil, err + } + repo.EnableProxy = repoInfo.EnableProxy + repo.RepoNamespace = repo.GetRepoNamespace() + } if len(build.JobCtx.Builds) == 0 { build.JobCtx.Builds = make([]*types.Repository, 0) } - build.JobCtx.BuildSteps = []*task.BuildStep{} + build.JobCtx.BuildSteps = []*taskmodels.BuildStep{} if module.Scripts != "" { - build.JobCtx.BuildSteps = append(build.JobCtx.BuildSteps, &task.BuildStep{BuildType: "shell", Scripts: module.Scripts}) + build.JobCtx.BuildSteps = append(build.JobCtx.BuildSteps, &taskmodels.BuildStep{BuildType: "shell", Scripts: module.Scripts}) } if module.PMDeployScripts != "" && build.ServiceType == setting.PMDeployType { @@ -1650,34 +2178,49 @@ func BuildModuleToSubTasks(moduleName, target, serviceName, productName string, } if len(module.SSHs) > 0 && build.ServiceType == setting.PMDeployType { - privateKeys := make([]*task.SSH, 0) + privateKeys := make([]*taskmodels.SSH, 0) for _, sshID := range module.SSHs { //私钥信息可能被更新，而构建中存储的信息是旧的，需要根据id获取最新的私钥信息 latestKeyInfo, err := commonrepo.NewPrivateKeyColl().Find(commonrepo.FindPrivateKeyOption{ID: sshID}) if err != nil || latestKeyInfo == nil { - log.Errorf("PrivateKey.Find failed, id:%s, err:%v", sshID, err) + log.Errorf("PrivateKey.Find failed, id:%s, err:%s", sshID, err) continue } - ssh := new(task.SSH) + ssh := new(taskmodels.SSH) ssh.Name = latestKeyInfo.Name ssh.UserName = latestKeyInfo.UserName ssh.IP = latestKeyInfo.IP + ssh.Port = latestKeyInfo.Port + if ssh.Port == 0 { + ssh.Port = setting.PMHostDefaultPort + } ssh.PrivateKey = latestKeyInfo.PrivateKey privateKeys = append(privateKeys, ssh) + delete(envHostInfos, sshID) } build.JobCtx.SSHs = privateKeys } + // Sync from the configuration in the environment + for _, privateKey := range envHostInfos { + build.JobCtx.SSHs = append(build.JobCtx.SSHs, &taskmodels.SSH{ + Name: privateKey.Name, + UserName: privateKey.UserName, + IP: privateKey.IP, + Port: privateKey.Port, + PrivateKey: privateKey.PrivateKey, + }) + } build.JobCtx.EnvVars = module.PreBuild.Envs - if len(module.PreBuild.Envs) == 0 { + if len(build.JobCtx.EnvVars) == 0 { build.JobCtx.EnvVars = make([]*commonmodels.KeyVal, 0) } - if len(envs) > 0 { + if len(args.Variables) > 0 { for _, envVar := range build.JobCtx.EnvVars { - for _, overwrite := range envs { + for _, overwrite := range args.Variables { if overwrite.Key == envVar.Key && overwrite.Value != setting.MaskValue { envVar.Value = overwrite.Value envVar.IsCredential = overwrite.IsCredential @@ -1692,15 +2235,23 @@ func BuildModuleToSubTasks(moduleName, target, serviceName, productName string, build.JobCtx.EnableProxy = module.PreBuild.EnableProxy if module.PostBuild != nil && module.PostBuild.DockerBuild != nil { - build.JobCtx.DockerBuildCtx = &task.DockerBuildCtx{ - WorkDir: module.PostBuild.DockerBuild.WorkDir, - DockerFile: module.PostBuild.DockerBuild.DockerFile, - BuildArgs: module.PostBuild.DockerBuild.BuildArgs, + dockerTemplateContent := "" + if module.PostBuild.DockerBuild.TemplateID != "" { + if dockerfileDetail, err := templ.GetDockerfileTemplateDetail(module.PostBuild.DockerBuild.TemplateID, log); err == nil { + dockerTemplateContent = dockerfileDetail.Content + } + } + build.JobCtx.DockerBuildCtx = &taskmodels.DockerBuildCtx{ + Source: module.PostBuild.DockerBuild.Source, + WorkDir: module.PostBuild.DockerBuild.WorkDir, + DockerFile: module.PostBuild.DockerBuild.DockerFile, + BuildArgs: module.PostBuild.DockerBuild.BuildArgs, + DockerTemplateContent: dockerTemplateContent, } } if module.PostBuild != nil && module.PostBuild.FileArchive != nil { - build.JobCtx.FileArchiveCtx = &task.FileArchiveCtx{ + build.JobCtx.FileArchiveCtx = &taskmodels.FileArchiveCtx{ FileLocation: module.PostBuild.FileArchive.FileLocation, } } @@ -1709,8 +2260,37 @@ func BuildModuleToSubTasks(moduleName, target, serviceName, productName string, build.JobCtx.PostScripts = module.PostBuild.Scripts } + if module.PostBuild != nil && module.PostBuild.ObjectStorageUpload != nil { + build.JobCtx.UploadEnabled = module.PostBuild.ObjectStorageUpload.Enabled + if module.PostBuild.ObjectStorageUpload.Enabled { + storageInfo, err := commonrepo.NewS3StorageColl().Find(module.PostBuild.ObjectStorageUpload.ObjectStorageID) + if err != nil { + log.Errorf("Failed to get basic storage info for uploading, the error is %s", err) + return nil, err + } + build.JobCtx.UploadStorageInfo = &types.ObjectStorageInfo{ + Endpoint: storageInfo.Endpoint, + AK: storageInfo.Ak, + SK: storageInfo.Sk, + Bucket: storageInfo.Bucket, + Insecure: storageInfo.Insecure, + Provider: storageInfo.Provider, + } + build.JobCtx.UploadInfo = module.PostBuild.ObjectStorageUpload.UploadDetail + } + } + build.JobCtx.Caches = module.Caches + if args.FileName != "" { + build.ArtifactInfo = &taskmodels.ArtifactInfo{ + URL: args.URL, + WorkflowName: args.WorkflowName, + TaskID: args.TaskID, + FileName: args.FileName, + } + } + bst, err := build.ToSubTask() if err != nil { return subTasks, e.ErrConvertSubTasks.AddErr(err) @@ -1721,20 +2301,57 @@ func BuildModuleToSubTasks(moduleName, target, serviceName, productName string, return subTasks, nil } -func ensurePipelineTask(pt *task.Task, envName string, log *zap.SugaredLogger) error { +func extractHost(privateKeys []*commonmodels.PrivateKey, ips, names sets.String, envHostInfos map[string]*commonmodels.PrivateKey) (sets.String, sets.String) { + for _, privateKey := range privateKeys { + ips.Insert(privateKey.IP) + names.Insert(privateKey.Name) + envHostInfos[privateKey.ID.Hex()] = privateKey + } + return ips, names +} + +func getServiceNaming(projectName, envName, serviceName string) (string, error) { + productInfo, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: projectName, + EnvName: envName, + }) + if err != nil { + return "", err + } + if productInfo.Source != setting.SourceFromHelm { + return "", nil + } + svcMap := productInfo.GetServiceMap() + pSvc, ok := svcMap[serviceName] + if !ok { + return "", fmt.Errorf("can't find service: %s in product: %s:%s", serviceName, projectName, envName) + } + templateSvc, err := commonrepo.NewServiceColl().Find(&commonrepo.ServiceFindOption{ + ServiceName: serviceName, + Revision: pSvc.Revision, + Type: setting.HelmDeployType, + ProductName: projectName, + }) + if err != nil { + return "", nil + } + return util.GeneReleaseName(templateSvc.GetReleaseNaming(), projectName, productInfo.Namespace, envName, serviceName), nil +} + +func ensurePipelineTask(taskOpt *taskmodels.TaskOpt, log *zap.SugaredLogger) error { var ( buildEnvs []*commonmodels.KeyVal ) // 验证 Subtask payload - err := validateSubTaskSetting(pt.PipelineName, pt.SubTasks) + err := validateSubTaskSetting(taskOpt.Task.PipelineName, taskOpt.Task.SubTasks) if err != nil { log.Errorf("Validate subtask setting failed: %+v", err) return err } //设置执行任务时参数 - for i, subTask := range pt.SubTasks { + for i, subTask := range taskOpt.Task.SubTasks { pre, err := base.ToPreview(subTask) if err != nil { @@ -1743,7 +2360,7 @@ func ensurePipelineTask(pt *task.Task, envName string, log *zap.SugaredLogger) e switch pre.TaskType { - case config.TaskBuild: + case config.TaskBuild, config.TaskArtifactDeploy, config.TaskBuildV3: t, err := base.ToBuildTask(subTask) fmtBuildsTask(t, log) if err != nil { @@ -1763,10 +2380,9 @@ func ensurePipelineTask(pt *task.Task, envName string, log *zap.SugaredLogger) e // } // } //} - // 设置Pipeline对应的服务名称 if t.ServiceName != "" { - pt.ServiceName = t.ServiceName + taskOpt.Task.ServiceName = t.ServiceName } // 设置 build 安装脚本 @@ -1777,10 +2393,17 @@ func ensurePipelineTask(pt *task.Task, envName string, log *zap.SugaredLogger) e } // 外部触发的pipeline - if pt.TaskCreator == setting.WebhookTaskCreator || pt.TaskCreator == setting.CronTaskCreator { - SetTriggerBuilds(t.JobCtx.Builds, pt.TaskArgs.Builds, log) + if taskOpt.Task.TaskCreator == setting.WebhookTaskCreator || taskOpt.Task.TaskCreator == setting.CronTaskCreator { + SetTriggerBuilds(t.JobCtx.Builds, taskOpt.Task.TaskArgs.Builds, log) } else { - setManunalBuilds(t.JobCtx.Builds, pt.TaskArgs.Builds, log) + setManunalBuilds(t.JobCtx.Builds, taskOpt.Task.TaskArgs.Builds, log) + } + + opt := &commonrepo.ProductFindOptions{EnvName: taskOpt.EnvName, Name: taskOpt.Task.ProductName} + exitedProd, err := commonrepo.NewProductColl().Find(opt) + if err != nil { + log.Errorf("can't find product by envName:%s error msg: %s", taskOpt.EnvName, err) + return e.ErrFindRegistry.AddDesc(err.Error()) } // 生成默认镜像tag后缀 @@ -1789,31 +2412,48 @@ func ensurePipelineTask(pt *task.Task, envName string, log *zap.SugaredLogger) e // 设置镜像名称 // 编译任务使用 t.JobCtx.Image // 注意: 其他任务从 pt.TaskArgs.Deploy.Image 获取, 必须要有编译任务 - reg, err := commonservice.FindDefaultRegistry(log) - if err != nil { - log.Errorf("can't find default candidate registry: %v", err) - return e.ErrFindRegistry.AddDesc(err.Error()) + var reg *commonmodels.RegistryNamespace + if len(exitedProd.RegistryID) > 0 { + reg, _, err = commonservice.FindRegistryById(exitedProd.RegistryID, true, log) + if err != nil { + log.Errorf("service.EnsureRegistrySecret: failed to find registry: %s error msg:%s", + exitedProd.RegistryID, err) + return e.ErrFindRegistry.AddDesc(err.Error()) + } + } else { + reg, _, err = commonservice.FindDefaultRegistry(true, log) + if err != nil { + log.Errorf("can't find default candidate registry: %s", err) + return e.ErrFindRegistry.AddDesc(err.Error()) + } } + t.JobCtx.Image = GetImage(reg, commonservice.ReleaseCandidate(t.JobCtx.Builds, taskOpt.Task.TaskID, taskOpt.Task.ProductName, t.ServiceName, taskOpt.EnvName, taskOpt.ImageName, "image")) + taskOpt.Task.TaskArgs.Deploy.Image = t.JobCtx.Image - t.JobCtx.Image = GetImage(reg, releaseCandidate(t, pt.TaskID, pt.ProductName, envName, "image")) - pt.TaskArgs.Deploy.Image = t.JobCtx.Image + if taskOpt.ServiceName != "" { + *taskOpt.ServiceInfos = append(*taskOpt.ServiceInfos, &taskmodels.ServiceInfo{ + ServiceName: taskOpt.ServiceName, + ServiceModule: t.ServiceName, + Image: t.JobCtx.Image, + }) + } - if pt.ConfigPayload != nil { - pt.ConfigPayload.Registry.Addr = reg.RegAddr - pt.ConfigPayload.Registry.AccessKey = reg.AccessKey - pt.ConfigPayload.Registry.SecretKey = reg.SecretKey - pt.ConfigPayload.Registry.Namespace = reg.Namespace + if taskOpt.Task.ConfigPayload != nil { + taskOpt.Task.ConfigPayload.Registry.Addr = reg.RegAddr + taskOpt.Task.ConfigPayload.Registry.AccessKey = reg.AccessKey + taskOpt.Task.ConfigPayload.Registry.SecretKey = reg.SecretKey + taskOpt.Task.ConfigPayload.Registry.Namespace = reg.Namespace } // 二进制文件名称 // 编译任务使用 t.JobCtx.PackageFile // 注意: 其他任务从 pt.TaskArgs.Deploy.PackageFile 获取, 必须要有编译任务 - t.JobCtx.PackageFile = GetPackageFile(releaseCandidate(t, pt.TaskID, pt.ProductName, envName, "tar")) - pt.TaskArgs.Deploy.PackageFile = t.JobCtx.PackageFile + t.JobCtx.PackageFile = GetPackageFile(commonservice.ReleaseCandidate(t.JobCtx.Builds, taskOpt.Task.TaskID, taskOpt.Task.ProductName, t.ServiceName, taskOpt.EnvName, taskOpt.ImageName, "tar")) + taskOpt.Task.TaskArgs.Deploy.PackageFile = t.JobCtx.PackageFile // 注入编译模块中用户定义环境变量 // 注意: 需要在pt.TaskArgs.Deploy设置完之后再设置环境变量 - t.JobCtx.EnvVars = append(t.JobCtx.EnvVars, prepareTaskEnvs(pt, log)...) + t.JobCtx.EnvVars = append(t.JobCtx.EnvVars, prepareTaskEnvs(taskOpt.Task, log)...) // 如果其他模块需要使用编译模块的环境变量进行渲染，需要设置buildEnvs buildEnvs = t.JobCtx.EnvVars @@ -1837,16 +2477,23 @@ func ensurePipelineTask(pt *task.Task, envName string, log *zap.SugaredLogger) e // } // } //} + registryRepo := reg.RegAddr + "/" + reg.Namespace + if reg.RegProvider == config.RegistryTypeAWS { + registryRepo = reg.RegAddr + } - t.DockerBuildStatus = &task.DockerBuildStatus{ + t.DockerBuildStatus = &taskmodels.DockerBuildStatus{ ImageName: t.JobCtx.Image, - RegistryRepo: reg.RegAddr + "/" + reg.Namespace, + RegistryRepo: registryRepo, } + t.UTStatus = &task.UTStatus{} t.StaticCheckStatus = &task.StaticCheckStatus{} t.BuildStatus = &task.BuildStatus{} - - pt.SubTasks[i], err = t.ToSubTask() + if taskOpt.IsWorkflowTask { + t.ServiceName = t.ServiceName + "_" + t.Service + } + taskOpt.Task.SubTasks[i], err = t.ToSubTask() if err != nil { return err @@ -1861,23 +2508,83 @@ func ensurePipelineTask(pt *task.Task, envName string, log *zap.SugaredLogger) e if t.Enabled { // 分析镜像名称 image := "" - for _, jenkinsBuildParams := range t.JenkinsBuildArgs.JenkinsBuildParams { + for i, jenkinsBuildParams := range t.JenkinsBuildArgs.JenkinsBuildParams { if jenkinsBuildParams.Name != "IMAGE" { continue } - if value, ok := jenkinsBuildParams.Value.(string); ok { + + if jenkinsBuildParams.AutoGenerate { + opt := &commonrepo.ProductFindOptions{EnvName: taskOpt.EnvName, Name: taskOpt.Task.ProductName} + exitedProd, err := commonrepo.NewProductColl().Find(opt) + if err != nil { + log.Errorf("can't find product by envName:%s error msg: %s", taskOpt.EnvName, err) + return e.ErrFindRegistry.AddDesc(err.Error()) + } + + var reg *commonmodels.RegistryNamespace + if len(exitedProd.RegistryID) > 0 { + reg, _, err = commonservice.FindRegistryById(exitedProd.RegistryID, true, log) + if err != nil { + log.Errorf("service.EnsureRegistrySecret: failed to find registry: %s error msg:%s", + exitedProd.RegistryID, err) + return e.ErrFindRegistry.AddDesc(err.Error()) + } + } else { + reg, _, err = commonservice.FindDefaultRegistry(true, log) + if err != nil { + log.Errorf("can't find default candidate registry: %s", err) + return e.ErrFindRegistry.AddDesc(err.Error()) + } + } + project, err := templaterepo.NewProductColl().Find(taskOpt.Task.ProductName) + if err != nil { + log.Errorf("find project err:%s", err) + return err + } + + if project.CustomImageRule != nil && project.CustomImageRule.JenkinsRule != "" { + va := commonservice.Variable{ + SERVICE: t.ServiceName, + TIMESTAMP: time.Now().Format("20060102150405"), + IMAGE_NAME: taskOpt.ImageName, + PROJECT: taskOpt.Task.ProductName, + ENV_NAME: taskOpt.EnvName, + TASK_ID: strconv.FormatInt(taskOpt.Task.TaskID, 10), + } + tm, err := gotempl.New("jenkins").Parse(project.CustomImageRule.JenkinsRule) + if err != nil { + log.Errorf("Parse template err:%s", err) + return err + } + var replaceRuleVariable = gotempl.Must(tm, err) + payload := bytes.NewBufferString("") + err = replaceRuleVariable.Execute(payload, va) + if err != nil { + log.Errorf("Execute template err:%s", err) + return err + } + image = GetImage(reg, payload.String()) + } else { + image = GetImage(reg, fmt.Sprintf("%s:%s", taskOpt.ImageName, time.Now().Format("20060102150405"))) + } + + t.JenkinsBuildArgs.JenkinsBuildParams[i].Value = image + break + } else if value, ok := jenkinsBuildParams.Value.(string); ok { image = value break } } - if image == "" || !strings.Contains(image, ":") { return &ImageIllegal{} } - pt.TaskArgs.Deploy.Image = image + taskOpt.Task.TaskArgs.Deploy.Image = image t.Image = image - pt.SubTasks[i], err = t.ToSubTask() + if taskOpt.IsWorkflowTask { + t.ServiceName = t.ServiceName + "_" + t.Service + } + taskOpt.Task.SubTasks[i], err = t.ToSubTask() if err != nil { return err } @@ -1889,15 +2596,19 @@ func ensurePipelineTask(pt *task.Task, envName string, log *zap.SugaredLogger) e return err } if t.Enabled { - if pt.TaskArgs == nil { - pt.TaskArgs = &commonmodels.TaskArgs{PipelineName: pt.WorkflowArgs.WorkflowName, TaskCreator: pt.WorkflowArgs.WorklowTaskCreator} + if taskOpt.Task.TaskArgs == nil { + taskOpt.Task.TaskArgs = &commonmodels.TaskArgs{PipelineName: taskOpt.Task.WorkflowArgs.WorkflowName, TaskCreator: taskOpt.Task.WorkflowArgs.WorkflowTaskCreator} } - registry := pt.ConfigPayload.RepoConfigs[pt.WorkflowArgs.RegistryID] - t.Image = fmt.Sprintf("%s/%s/%s", util.GetURLHostName(registry.RegAddr), registry.Namespace, t.Image) - pt.TaskArgs.Deploy.Image = t.Image - t.RegistryID = pt.WorkflowArgs.RegistryID + registry := taskOpt.Task.ConfigPayload.RepoConfigs[taskOpt.Task.WorkflowArgs.RegistryID] + if registry.RegProvider == config.RegistryTypeAWS { + t.Image = fmt.Sprintf("%s/%s", util.TrimURLScheme(registry.RegAddr), t.Image) + } else { + t.Image = fmt.Sprintf("%s/%s/%s", util.TrimURLScheme(registry.RegAddr), registry.Namespace, t.Image) + } + taskOpt.Task.TaskArgs.Deploy.Image = t.Image + t.RegistryID = taskOpt.Task.WorkflowArgs.RegistryID - pt.SubTasks[i], err = t.ToSubTask() + taskOpt.Task.SubTasks[i], err = t.ToSubTask() if err != nil { return err } @@ -1909,7 +2620,7 @@ func ensurePipelineTask(pt *task.Task, envName string, log *zap.SugaredLogger) e return err } if t.Enabled { - t.Image = pt.TaskArgs.Deploy.Image + t.Image = taskOpt.Task.TaskArgs.Deploy.Image // 使用环境变量KEY渲染 docker build的各个参数 for _, env := range buildEnvs { @@ -1920,17 +2631,11 @@ func ensurePipelineTask(pt *task.Task, envName string, log *zap.SugaredLogger) e } } - err = SetCandidateRegistry(pt.ConfigPayload, log) - if err != nil { - return err - } - - pt.SubTasks[i], err = t.ToSubTask() + taskOpt.Task.SubTasks[i], err = t.ToSubTask() if err != nil { return err } } - case config.TaskTestingV2: t, err := base.ToTestingTask(subTask) if err != nil { @@ -1945,26 +2650,21 @@ func ensurePipelineTask(pt *task.Task, envName string, log *zap.SugaredLogger) e // Iterate test jobctx builds, and replace it if params specified from task. // 外部触发的pipeline - if pt.TaskCreator == setting.WebhookTaskCreator || pt.TaskCreator == setting.CronTaskCreator { - SetTriggerBuilds(t.JobCtx.Builds, pt.TaskArgs.Test.Builds, log) + if taskOpt.Task.TaskCreator == setting.WebhookTaskCreator || taskOpt.Task.TaskCreator == setting.CronTaskCreator { + SetTriggerBuilds(t.JobCtx.Builds, taskOpt.Task.TaskArgs.Test.Builds, log) } else { - setManunalBuilds(t.JobCtx.Builds, pt.TaskArgs.Test.Builds, log) - } - - err = SetCandidateRegistry(pt.ConfigPayload, log) - if err != nil { - return err + setManunalBuilds(t.JobCtx.Builds, taskOpt.Task.TaskArgs.Test.Builds, log) } //use log path - if pt.ServiceName == "" { - pt.ServiceName = t.TestName + if taskOpt.Task.ServiceName == "" { + taskOpt.Task.ServiceName = t.TestName } // 设置敏感信息 t.JobCtx.EnvVars = append(t.JobCtx.EnvVars, envs...) - pt.SubTasks[i], err = t.ToSubTask() + taskOpt.Task.SubTasks[i], err = t.ToSubTask() if err != nil { return err } @@ -1977,8 +2677,8 @@ func ensurePipelineTask(pt *task.Task, envName string, log *zap.SugaredLogger) e } if t.Enabled { - t.SetNamespace(pt.TaskArgs.Deploy.Namespace) - image, err := validateServiceContainer(t.EnvName, t.ProductName, t.ServiceName, t.ContainerName) + t.SetNamespace(taskOpt.Task.TaskArgs.Deploy.Namespace) + image, err := getImageInfoFromWorkload(t.EnvName, t.ProductName, t.ServiceName, t.ContainerName) if err != nil { log.Error(err) return err @@ -1986,7 +2686,7 @@ func ensurePipelineTask(pt *task.Task, envName string, log *zap.SugaredLogger) e t.SetImage(image) - pt.SubTasks[i], err = t.ToSubTask() + taskOpt.Task.SubTasks[i], err = t.ToSubTask() if err != nil { return err } @@ -2001,27 +2701,26 @@ func ensurePipelineTask(pt *task.Task, envName string, log *zap.SugaredLogger) e if t.Enabled { // 从创建任务payload设置容器部署 - t.SetImage(pt.TaskArgs.Deploy.Image) - t.SetNamespace(pt.TaskArgs.Deploy.Namespace) + t.SetImage(taskOpt.Task.TaskArgs.Deploy.Image) + t.SetNamespace(taskOpt.Task.TaskArgs.Deploy.Namespace) - _, err := validateServiceContainer(t.EnvName, t.ProductName, t.ServiceName, t.ContainerName) - if err != nil { - log.Error(err) - return err + containerName := t.ContainerName + if taskOpt.IsWorkflowTask { + containerName = strings.TrimSuffix(containerName, "_"+t.ServiceName) } // Task creator can be webhook trigger or cronjob trigger or validated user // Validated user includes both that user is granted write permission or user is the owner of this product - if pt.TaskCreator == setting.WebhookTaskCreator || - pt.TaskCreator == setting.CronTaskCreator || - IsProductAuthed(pt.TaskCreator, t.Namespace, pt.ProductName, config.ProductWritePermission, log) { - log.Infof("Validating permission passed. product:%s, owner:%s, task executed by: %s", pt.ProductName, t.Namespace, pt.TaskCreator) + if taskOpt.Task.TaskCreator == setting.WebhookTaskCreator || + taskOpt.Task.TaskCreator == setting.CronTaskCreator || + IsProductAuthed(taskOpt.Task.TaskCreator, t.Namespace, taskOpt.Task.ProductName, config.ProductWritePermission, log) { + log.Infof("Validating permission passed. product:%s, owner:%s, task executed by: %s", taskOpt.Task.ProductName, t.Namespace, taskOpt.Task.TaskCreator) } else { - log.Errorf("permission denied. product:%s, owner:%s, task executed by: %s", pt.ProductName, t.Namespace, pt.TaskCreator) + log.Errorf("permission denied. product:%s, owner:%s, task executed by: %s", taskOpt.Task.ProductName, t.Namespace, taskOpt.Task.TaskCreator) return errors.New(e.ProductAccessDeniedErrMsg) } - pt.SubTasks[i], err = t.ToSubTask() + taskOpt.Task.SubTasks[i], err = t.ToSubTask() if err != nil { return err } @@ -2034,15 +2733,15 @@ func ensurePipelineTask(pt *task.Task, envName string, log *zap.SugaredLogger) e return err } if task.Enabled { - task.SetPackageFile(pt.TaskArgs.Deploy.PackageFile) + task.SetPackageFile(taskOpt.Task.TaskArgs.Deploy.PackageFile) - if pt.TeamName == "" { - task.ProductName = pt.ProductName + if taskOpt.Task.TeamName == "" { + task.ProductName = taskOpt.Task.ProductName } else { - task.ProductName = pt.TeamName + task.ProductName = taskOpt.Task.TeamName } - task.ServiceName = pt.ServiceName + task.ServiceName = taskOpt.Task.ServiceName if task.DestStorageURL == "" { var storage *commonmodels.S3Storage @@ -2064,7 +2763,7 @@ func ensurePipelineTask(pt *task.Task, envName string, log *zap.SugaredLogger) e return err } } - pt.SubTasks[i], err = task.ToSubTask() + taskOpt.Task.SubTasks[i], err = task.ToSubTask() if err != nil { return err } @@ -2079,11 +2778,11 @@ func ensurePipelineTask(pt *task.Task, envName string, log *zap.SugaredLogger) e if t.Enabled { // 从创建任务payload设置线上镜像分发任务 - t.SetImage(pt.TaskArgs.Deploy.Image) + t.SetImage(taskOpt.Task.TaskArgs.Deploy.Image) // 兼容老的任务配置 if len(t.Releases) == 0 { found := false - for id, v := range pt.ConfigPayload.RepoConfigs { + for id, v := range taskOpt.Task.ConfigPayload.RepoConfigs { if v.Namespace == t.ImageRepo { t.Releases = append(t.Releases, commonmodels.RepoImage{RepoID: id}) found = true @@ -2097,23 +2796,54 @@ func ensurePipelineTask(pt *task.Task, envName string, log *zap.SugaredLogger) e var repos []commonmodels.RepoImage for _, repoImage := range t.Releases { - if v, ok := pt.ConfigPayload.RepoConfigs[repoImage.RepoID]; ok { - repoImage.Name = util.ReplaceRepo(pt.TaskArgs.Deploy.Image, v.RegAddr, v.Namespace) - repoImage.Host = util.GetURLHostName(v.RegAddr) + if v, ok := taskOpt.Task.ConfigPayload.RepoConfigs[repoImage.RepoID]; ok { + repoImage.Name = util.ReplaceRepo(taskOpt.Task.TaskArgs.Deploy.Image, v.RegAddr, v.Namespace) + repoImage.Host = util.TrimURLScheme(v.RegAddr) repoImage.Namespace = v.Namespace repos = append(repos, repoImage) } } + var distributes []*taskmodels.DistributeInfo + for _, distribute := range t.DistributeInfo { + d := &taskmodels.DistributeInfo{ + DeployEnabled: distribute.DeployEnabled, + DeployEnv: distribute.DeployEnv, + DeployNamespace: distribute.DeployNamespace, + DeployContainerName: taskOpt.Task.ServiceName, // This is a match for target.Name + DeployServiceName: taskOpt.ServiceName, // this is a match for target.ServiceName + DeployServiceType: distribute.DeployServiceType, + DeployClusterID: distribute.DeployClusterID, + RepoID: distribute.RepoID, + } + if distribute.DeployEnabled { + releaseNaming, err := getServiceNaming(taskOpt.Task.ProductName, distribute.DeployEnv, taskOpt.ServiceName) + if err != nil { + return fmt.Errorf("failed to find release naming for service: %s, err: %s", taskOpt.ServiceName, err) + } + d.ReleaseName = releaseNaming + } + if v, ok := taskOpt.Task.ConfigPayload.RepoConfigs[distribute.RepoID]; ok { + d.Image = util.ReplaceRepo(taskOpt.Task.TaskArgs.Deploy.Image, v.RegAddr, v.Namespace) + } + distributes = append(distributes, d) + } + t.Releases = repos + t.DistributeInfo = distributes if len(t.Releases) == 0 { t.Enabled = false } else { t.ImageRelease = t.Releases[0].Name } - pt.SubTasks[i], err = t.ToSubTask() + if len(t.DistributeInfo) == 0 { + t.Enabled = false + } + + taskOpt.Task.SubTasks[i], err = t.ToSubTask() if err != nil { + log.Errorf("release task to subtask error: %s", err) return err } } @@ -2127,13 +2857,13 @@ func ensurePipelineTask(pt *task.Task, envName string, log *zap.SugaredLogger) e FmtBuilds(t.Builds, log) if t.Enabled { // 外部触发的pipeline - if pt.TaskCreator == setting.WebhookTaskCreator || pt.TaskCreator == setting.CronTaskCreator { - SetTriggerBuilds(t.Builds, pt.TaskArgs.Builds, log) + if taskOpt.Task.TaskCreator == setting.WebhookTaskCreator || taskOpt.Task.TaskCreator == setting.CronTaskCreator { + SetTriggerBuilds(t.Builds, taskOpt.Task.TaskArgs.Builds, log) } else { - setManunalBuilds(t.Builds, pt.TaskArgs.Builds, log) + setManunalBuilds(t.Builds, taskOpt.Task.TaskArgs.Builds, log) } - pt.SubTasks[i], err = t.ToSubTask() + taskOpt.Task.SubTasks[i], err = t.ToSubTask() if err != nil { return err } @@ -2146,14 +2876,27 @@ func ensurePipelineTask(pt *task.Task, envName string, log *zap.SugaredLogger) e } if t.Enabled { - t.SetImageName(pt.TaskArgs.Deploy.Image) - pt.SubTasks[i], err = t.ToSubTask() + t.SetImageName(taskOpt.Task.TaskArgs.Deploy.Image) + taskOpt.Task.SubTasks[i], err = t.ToSubTask() + if err != nil { + return err + } + } + case config.TaskDistribute, config.TaskArtifactPackage: + // do nothing + case config.TaskTrigger: + t, err := base.ToTriggerTask(subTask) + if err != nil { + log.Error(err) + return err + } + + if t.Enabled { + taskOpt.Task.SubTasks[i], err = t.ToSubTask() if err != nil { return err } } - case config.TaskDistribute: - // 为了兼容历史数据类型，目前什么都不用做，避免出错 default: return e.NewErrInvalidTaskType(string(pre.TaskType)) } diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/workflow_task_v3.go b/pkg/microservice/aslan/core/workflow/service/workflow/workflow_task_v3.go new file mode 100644 index 0000000000000000000000000000000000000000..62629b3964960c28f9a99588573aff9f99bd7ee5 --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/service/workflow/workflow_task_v3.go @@ -0,0 +1,260 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package workflow + +import ( + "fmt" + "sort" + + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3" + "github.com/koderover/zadig/pkg/setting" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +type TaskResp struct { + Name string `json:"name"` + TaskID int64 `json:"task_id"` +} + +func CreateWorkflowTaskV3(args *commonmodels.WorkflowV3Args, username, reqID string, log *zap.SugaredLogger) (*TaskResp, error) { + workflowV3, err := commonrepo.NewWorkflowV3Coll().GetByID(args.ID) + if err != nil { + log.Errorf("workflowV3.Find %s error: %s", args.Name, err) + return nil, e.ErrCreateTask.AddDesc(e.FindPipelineErrMsg) + } + + nextTaskID, err := commonrepo.NewCounterColl().GetNextSeq(fmt.Sprintf(setting.WorkflowTaskV3Fmt, args.Name)) + if err != nil { + log.Errorf("Counter.GetNextSeq error: %s", err) + return nil, e.ErrGetCounter.AddDesc(err.Error()) + } + + // 自定义基础镜像的镜像名称可能会被更新，需要使用ID获取最新的镜像名称 + for i, subTask := range workflowV3.SubTasks { + pre, err := base.ToPreview(subTask) + if err != nil { + log.Errorf("subTask.ToPreview error: %s", err) + continue + } + switch pre.TaskType { + case config.TaskBuildV3: + build, err := base.ToBuildTask(subTask) + if err != nil || build == nil { + log.Errorf("subTask.ToBuildTask error: %s", err) + continue + } + if build.ImageID == "" { + continue + } + basicImage, err := commonrepo.NewBasicImageColl().Find(build.ImageID) + if err != nil { + log.Errorf("BasicImage.Find failed, id:%s, err:%s", build.ImageID, err) + continue + } + build.BuildOS = basicImage.Value + + // 创建任务时可以临时编辑环境变量，需要将pipeline中的环境变量更新成新的值。 + if args.BuildArgs != nil { + build.JobCtx.EnvVars = args.BuildArgs + } + build.ServiceName = fmt.Sprintf("%s-job", args.Name) + workflowV3.SubTasks[i], err = build.ToSubTask() + if err != nil { + log.Errorf("build.ToSubTask error: %s", err) + continue + } + case config.TaskTrigger: + trigger, err := base.ToTriggerTask(subTask) + if err != nil || trigger == nil { + log.Errorf("subTask.ToTriggerTask error: %s", err) + continue + } + workflowV3.SubTasks[i], err = trigger.ToSubTask() + } + } + + var defaultStorageURI string + if defaultS3, err := s3.FindDefaultS3(); err == nil { + defaultStorageURI, err = defaultS3.GetEncryptedURL() + if err != nil { + return nil, e.ErrS3Storage.AddErr(err) + } + } + + pt := &task.Task{ + TaskID: nextTaskID, + ProductName: args.ProjectName, + PipelineName: args.Name, + Type: config.WorkflowTypeV3, + TaskCreator: username, + ReqID: reqID, + TaskArgs: &commonmodels.TaskArgs{Builds: args.Builds, BuildArgs: args.BuildArgs}, + Status: config.StatusCreated, + SubTasks: workflowV3.SubTasks, + ConfigPayload: commonservice.GetConfigPayload(0), + StorageURI: defaultStorageURI, + } + + sort.Sort(ByTaskKind(pt.SubTasks)) + + if err := ensurePipelineTask(&task.TaskOpt{ + Task: pt, + }, log); err != nil { + log.Errorf("Service.ensurePipelineTask failed %v %s", args, err) + if err, ok := err.(*ContainerNotFound); ok { + return nil, e.NewWithExtras( + e.ErrCreateTaskFailed, + "container doesn't exists", map[string]interface{}{ + "productName": err.ProductName, + "envName": err.EnvName, + "serviceName": err.ServiceName, + "containerName": err.Container, + }) + } + + return nil, e.ErrCreateTask.AddDesc(err.Error()) + } + + if len(pt.SubTasks) <= 0 { + return nil, e.ErrCreateTask.AddDesc(e.PipelineSubTaskNotFoundErrMsg) + } + + // send to queue to execute task + if err := CreateTask(pt); err != nil { + log.Error(err) + return nil, e.ErrCreateTask + } + + resp := &TaskResp{ + Name: args.Name, + TaskID: nextTaskID, + } + return resp, nil +} + +func RestartWorkflowTaskV3(userName string, taskID int64, workflowName string, typeString config.PipelineType, log *zap.SugaredLogger) error { + t, err := commonrepo.NewTaskColl().Find(taskID, workflowName, typeString) + if err != nil { + log.Errorf("[%d:%s] find workflow task v3 error: %s", taskID, workflowName, err) + return e.ErrRestartTask.AddDesc(e.FindPipelineTaskErrMsg) + } + + // 不重试已经成功的workflow task + if t.Status == config.StatusRunning || t.Status == config.StatusPassed { + log.Errorf("cannot restart running or passed task. Status: %v", t.Status) + return e.ErrRestartTask.AddDesc(e.RestartPassedTaskErrMsg) + } + + t.IsRestart = true + t.Status = config.StatusCreated + t.TaskCreator = userName + if err := UpdateTask(t); err != nil { + log.Errorf("update workflow task v3 error: %s", err) + return e.ErrRestartTask.AddDesc(e.UpdatePipelineTaskErrMsg) + } + return nil +} + +// ListWorkflowTasksV3Result 工作流任务分页信息 +func ListWorkflowTasksV3Result(name string, typeString config.PipelineType, maxResult, startAt int, log *zap.SugaredLogger) (*TaskResult, error) { + ret := &TaskResult{MaxResult: maxResult, StartAt: startAt} + var err error + ret.Data, err = commonrepo.NewTaskColl().List(&commonrepo.ListTaskOption{PipelineName: name, Limit: maxResult, Skip: startAt, Detail: true, Type: typeString}) + if err != nil { + log.Errorf("PipelineTaskV2.List error: %s", err) + return ret, e.ErrListTasks + } + + pipelineList := []string{name} + ret.Total, err = commonrepo.NewTaskColl().Count(&commonrepo.CountTaskOption{PipelineNames: pipelineList, Type: typeString}) + if err != nil { + log.Errorf("workflowTaskV3.List error: %s", err) + return ret, e.ErrCountTasks + } + return ret, nil +} + +func GetWorkflowTaskV3(taskID int64, pipelineName string, typeString config.PipelineType, log *zap.SugaredLogger) (*task.Task, error) { + resp, err := commonrepo.NewTaskColl().Find(taskID, pipelineName, typeString) + if err != nil { + log.Errorf("[%d:%s] PipelineTaskV2.Find error: %s", taskID, pipelineName, err) + return resp, e.ErrGetTask + } + + CleanWorkflow3(resp) + return resp, nil +} + +func GetWorkflowTaskV3Callback(taskID int64, pipelineName string, logger *zap.SugaredLogger) (*commonmodels.CallbackRequest, error) { + return commonrepo.NewCallbackRequestColl().Find(&commonrepo.CallbackFindOption{ + TaskID: taskID, + PipelineName: pipelineName, + }) +} + +func CleanWorkflow3(task *task.Task) { + task.ConfigPayload = nil + task.TaskArgs = nil + + EnsureSubTasksV3Resp(task.SubTasks) + for _, stage := range task.Stages { + ensureSubStageV3Resp(stage) + } +} + +// EnsureSubTasksResp 确保SubTask中敏感信息和其他不必要信息不返回给前端 +func EnsureSubTasksV3Resp(subTasks []map[string]interface{}) { + for i, subTask := range subTasks { + pre, err := base.ToPreview(subTask) + if err != nil { + continue + } + + switch pre.TaskType { + case config.TaskBuildV3: + if newTask, err := ensureBuildTask(subTask); err == nil { + subTasks[i] = newTask + } + } + + } +} + +func ensureSubStageV3Resp(stage *commonmodels.Stage) { + subTasks := stage.SubTasks + for key, subTask := range subTasks { + pre, err := base.ToPreview(subTask) + if err != nil { + continue + } + + switch pre.TaskType { + case config.TaskBuildV3: + if newTask, err := ensureBuildTask(subTask); err == nil { + subTasks[key] = newTask + } + } + } +} diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/workflow_task_v4.go b/pkg/microservice/aslan/core/workflow/service/workflow/workflow_task_v4.go new file mode 100644 index 0000000000000000000000000000000000000000..76188e85222377001efe087834ebbcf35fbaaa8b --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/service/workflow/workflow_task_v4.go @@ -0,0 +1,397 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package workflow + +import ( + "fmt" + "time" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/workflowcontroller" + jobctl "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow/job" + "github.com/koderover/zadig/pkg/setting" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/types" + stepspec "github.com/koderover/zadig/pkg/types/step" + "go.uber.org/zap" +) + +type CreateTaskV4Resp struct { + ProjectName string `json:"project_name"` + WorkflowName string `json:"workflow_name"` + TaskID int64 `json:"task_id"` +} + +type WorkflowTaskPreview struct { + TaskID int64 `bson:"task_id" json:"task_id"` + WorkflowName string `bson:"workflow_name" json:"workflow_name"` + Status config.Status `bson:"status" json:"status,omitempty"` + TaskCreator string `bson:"task_creator" json:"task_creator,omitempty"` + TaskRevoker string `bson:"task_revoker,omitempty" json:"task_revoker,omitempty"` + CreateTime int64 `bson:"create_time" json:"create_time,omitempty"` + StartTime int64 `bson:"start_time" json:"start_time,omitempty"` + EndTime int64 `bson:"end_time" json:"end_time,omitempty"` + Stages []*StageTaskPreview `bson:"stages" json:"stages"` + ProjectName string `bson:"project_name" json:"project_name"` + Error string `bson:"error,omitempty" json:"error,omitempty"` + IsRestart bool `bson:"is_restart" json:"is_restart"` +} + +type StageTaskPreview struct { + Name string `bson:"name" json:"name"` + Status config.Status `bson:"status" json:"status"` + StartTime int64 `bson:"start_time" json:"start_time,omitempty"` + EndTime int64 `bson:"end_time" json:"end_time,omitempty"` + Parallel bool `bson:"parallel" json:"parallel"` + Approval *commonmodels.Approval `bson:"approval" json:"approval"` + Jobs []*JobTaskPreview `bson:"jobs" json:"jobs"` +} + +type JobTaskPreview struct { + Name string `bson:"name" json:"name"` + JobType string `bson:"type" json:"type"` + Status config.Status `bson:"status" json:"status"` + StartTime int64 `bson:"start_time" json:"start_time,omitempty"` + EndTime int64 `bson:"end_time" json:"end_time,omitempty"` + Error string `bson:"error" json:"error"` + Spec interface{} `bson:"spec" json:"spec"` +} + +type ZadigBuildJobSpec struct { + Repos []*types.Repository `bson:"repos" json:"repos"` + Image string `bson:"image" json:"image"` + ServiceName string `bson:"service_name" json:"service_name"` + ServiceModule string `bson:"service_module" json:"service_module"` +} + +type ZadigDeployJobSpec struct { + Env string `bson:"env" json:"env"` + ServiceAndImages []*ServiceAndImage `bson:"service_and_images" json:"service_and_images"` +} + +type ServiceAndImage struct { + ServiceName string `bson:"service_name" json:"service_name"` + ServiceModule string `bson:"service_module" json:"service_module"` + Image string `bson:"image" json:"image"` +} + +func GetWorkflowv4Preset(encryptedKey, workflowName string, log *zap.SugaredLogger) (*commonmodels.WorkflowV4, error) { + workflow, err := commonrepo.NewWorkflowV4Coll().Find(workflowName) + if err != nil { + log.Errorf("cannot find workflow %s, the error is: %v", workflowName, err) + return nil, e.ErrFindWorkflow.AddDesc(err.Error()) + } + for _, stage := range workflow.Stages { + for _, job := range stage.Jobs { + if err := jobctl.SetPreset(job, workflow); err != nil { + log.Errorf("cannot get workflow %s preset, the error is: %v", workflowName, err) + return nil, e.ErrFindWorkflow.AddDesc(err.Error()) + } + } + } + if err := ensureWorkflowV4Resp(encryptedKey, workflow, log); err != nil { + return workflow, err + } + return workflow, nil +} + +func CreateWorkflowTaskV4(user string, workflow *commonmodels.WorkflowV4, log *zap.SugaredLogger) (*CreateTaskV4Resp, error) { + resp := &CreateTaskV4Resp{ + ProjectName: workflow.Project, + WorkflowName: workflow.Name, + } + if err := LintWorkflowV4(workflow, log); err != nil { + return resp, err + } + workflowTask := &commonmodels.WorkflowTask{} + nextTaskID, err := commonrepo.NewCounterColl().GetNextSeq(fmt.Sprintf(setting.WorkflowTaskV4Fmt, workflow.Name)) + if err != nil { + log.Errorf("Counter.GetNextSeq error: %v", err) + return resp, e.ErrGetCounter.AddDesc(err.Error()) + } + resp.TaskID = nextTaskID + + workflowTask.TaskID = nextTaskID + workflowTask.TaskCreator = user + workflowTask.TaskRevoker = user + workflowTask.CreateTime = time.Now().Unix() + workflowTask.WorkflowName = workflow.Name + workflowTask.ProjectName = workflow.Project + workflowTask.KeyVals = workflow.KeyVals + workflowTask.MultiRun = workflow.MultiRun + + for _, stage := range workflow.Stages { + stageTask := &commonmodels.StageTask{ + Name: stage.Name, + Parallel: stage.Parallel, + Approval: stage.Approval, + } + for _, job := range stage.Jobs { + // TODO: move this logic to job controller + if job.JobType == config.JobZadigBuild { + if err := setZadigBuildRepos(job, log); err != nil { + log.Errorf("zadig build job set build info error: %v", err) + return resp, e.ErrCreateTask.AddDesc(err.Error()) + } + } + if job.JobType == config.JobFreestyle { + if err := setFreeStyleRepos(job, log); err != nil { + log.Errorf("freestyle job set build info error: %v", err) + return resp, e.ErrCreateTask.AddDesc(err.Error()) + } + } + + jobs, err := jobctl.ToJobs(job, workflow, nextTaskID) + if err != nil { + log.Errorf("cannot create workflow %s, the error is: %v", workflow.Name, err) + return resp, e.ErrCreateTask.AddDesc(err.Error()) + } + stageTask.Jobs = append(stageTask.Jobs, jobs...) + } + if len(stageTask.Jobs) > 0 { + workflowTask.Stages = append(workflowTask.Stages, stageTask) + } + } + + if err := workflowTaskLint(workflowTask, log); err != nil { + return resp, err + } + + workflowTask.WorkflowArgs = workflow + + if err := workflowcontroller.CreateTask(workflowTask); err != nil { + log.Errorf("create workflow task error: %v", err) + return resp, e.ErrCreateTask.AddDesc(err.Error()) + } + return resp, nil +} + +func CloneWorkflowTaskV4(workflowName string, taskID int64, logger *zap.SugaredLogger) (*commonmodels.WorkflowV4, error) { + task, err := commonrepo.NewworkflowTaskv4Coll().Find(workflowName, taskID) + if err != nil { + logger.Errorf("find workflowTaskV4 error: %s", err) + return nil, e.ErrGetTask.AddErr(err) + } + return task.WorkflowArgs, nil +} + +func UpdateWorkflowTaskV4(id string, workflowTask *commonmodels.WorkflowTask, logger *zap.SugaredLogger) error { + err := commonrepo.NewworkflowTaskv4Coll().Update( + id, + workflowTask, + ) + if err != nil { + logger.Errorf("update workflowTaskV4 error: %s", err) + return e.ErrCreateTask.AddErr(err) + } + return nil +} + +func ListWorkflowTaskV4(workflowName string, pageNum, pageSize int64, logger *zap.SugaredLogger) ([]*commonmodels.WorkflowTask, int64, error) { + resp, total, err := commonrepo.NewworkflowTaskv4Coll().List(&commonrepo.ListWorkflowTaskV4Option{WorkflowName: workflowName, Limit: int(pageSize), Skip: int((pageNum - 1) * pageSize)}) + if err != nil { + logger.Errorf("list workflowTaskV4 error: %s", err) + return resp, total, err + } + return resp, total, nil +} + +func CancelWorkflowTaskV4(userName, workflowName string, taskID int64, logger *zap.SugaredLogger) error { + if err := workflowcontroller.CancelWorkflowTask(userName, workflowName, taskID, logger); err != nil { + logger.Errorf("cancel workflowTaskV4 error: %s", err) + return e.ErrCancelTask.AddErr(err) + } + return nil +} + +func GetWorkflowTaskV4(workflowName string, taskID int64, logger *zap.SugaredLogger) (*WorkflowTaskPreview, error) { + task, err := commonrepo.NewworkflowTaskv4Coll().Find(workflowName, taskID) + if err != nil { + logger.Errorf("find workflowTaskV4 error: %s", err) + return nil, err + } + resp := &WorkflowTaskPreview{ + TaskID: task.TaskID, + WorkflowName: task.WorkflowName, + ProjectName: task.ProjectName, + Status: task.Status, + TaskCreator: task.TaskCreator, + TaskRevoker: task.TaskRevoker, + CreateTime: task.CreateTime, + StartTime: task.StartTime, + EndTime: task.EndTime, + Error: task.Error, + IsRestart: task.IsRestart, + } + for _, stage := range task.Stages { + resp.Stages = append(resp.Stages, &StageTaskPreview{ + Name: stage.Name, + Status: stage.Status, + StartTime: stage.StartTime, + EndTime: stage.EndTime, + Parallel: stage.Parallel, + Approval: stage.Approval, + Jobs: jobsToJobPreviews(stage.Jobs), + }) + } + return resp, nil +} + +func ApproveStage(workflowName, stageName, userName, userID, comment string, taskID int64, approve bool, logger *zap.SugaredLogger) error { + if workflowName == "" || stageName == "" || taskID == 0 { + errMsg := fmt.Sprintf("can not find approved workflow: %s, taskID: %d,stage: %s", workflowName, taskID, stageName) + logger.Error(errMsg) + return e.ErrApproveTask.AddDesc(errMsg) + } + if err := workflowcontroller.ApproveStage(workflowName, stageName, userName, userID, comment, taskID, approve); err != nil { + logger.Error(err) + return e.ErrApproveTask.AddErr(err) + } + return nil +} + +func jobsToJobPreviews(jobs []*commonmodels.JobTask) []*JobTaskPreview { + resp := []*JobTaskPreview{} + for _, job := range jobs { + jobPreview := &JobTaskPreview{ + Name: job.Name, + Status: job.Status, + StartTime: job.StartTime, + EndTime: job.EndTime, + Error: job.Error, + JobType: job.JobType, + } + switch job.JobType { + case string(config.FreestyleType): + fallthrough + case string(config.JobZadigBuild): + spec := ZadigBuildJobSpec{} + for _, arg := range job.Properties.Envs { + if arg.Key == "IMAGE" { + spec.Image = arg.Value + continue + } + if arg.Key == "SERVICE" { + spec.ServiceName = arg.Value + continue + } + if arg.Key == "SERVICE_MODULE" { + spec.ServiceModule = arg.Value + continue + } + } + for _, step := range job.Steps { + if step.StepType == config.StepGit { + stepSpec := &stepspec.StepGitSpec{} + commonmodels.IToi(step.Spec, &stepSpec) + spec.Repos = stepSpec.Repos + continue + } + } + jobPreview.Spec = spec + case string(config.JobZadigDeploy): + spec := ZadigDeployJobSpec{} + for _, step := range job.Steps { + if step.StepType == config.StepDeploy { + stepSpec := &stepspec.StepDeploySpec{} + commonmodels.IToi(step.Spec, &stepSpec) + spec.Env = stepSpec.Env + spec.ServiceAndImages = append(spec.ServiceAndImages, &ServiceAndImage{ + ServiceName: stepSpec.ServiceName, + ServiceModule: stepSpec.ServiceModule, + Image: stepSpec.Image, + }) + continue + } + if step.StepType == config.StepHelmDeploy { + stepSpec := &stepspec.StepHelmDeploySpec{} + commonmodels.IToi(step.Spec, &stepSpec) + spec.Env = stepSpec.Env + for _, imageAndmodule := range stepSpec.ImageAndModules { + spec.ServiceAndImages = append(spec.ServiceAndImages, &ServiceAndImage{ + ServiceName: stepSpec.ServiceName, + ServiceModule: imageAndmodule.ServiceModule, + Image: imageAndmodule.Image, + }) + } + continue + } + } + jobPreview.Spec = spec + case string(config.JobPlugin): + jobPreview.Spec = job.Plugin + default: + jobPreview.Spec = job.Steps + } + resp = append(resp, jobPreview) + } + return resp +} + +func setZadigBuildRepos(job *commonmodels.Job, logger *zap.SugaredLogger) error { + spec := &commonmodels.ZadigBuildJobSpec{} + if err := commonmodels.IToi(job.Spec, spec); err != nil { + return err + } + for _, build := range spec.ServiceAndBuilds { + if err := setManunalBuilds(build.Repos, build.Repos, logger); err != nil { + return err + } + } + job.Spec = spec + return nil +} + +func setFreeStyleRepos(job *commonmodels.Job, logger *zap.SugaredLogger) error { + spec := &commonmodels.FreestyleJobSpec{} + if err := commonmodels.IToi(job.Spec, spec); err != nil { + return err + } + for _, step := range spec.Steps { + if step.StepType != config.StepGit { + continue + } + stepSpec := &stepspec.StepGitSpec{} + if err := commonmodels.IToi(step.Spec, stepSpec); err != nil { + return err + } + if err := setManunalBuilds(stepSpec.Repos, stepSpec.Repos, logger); err != nil { + return err + } + step.Spec = stepSpec + } + job.Spec = spec + return nil +} + +func workflowTaskLint(workflowTask *commonmodels.WorkflowTask, logger *zap.SugaredLogger) error { + if len(workflowTask.Stages) <= 0 { + errMsg := fmt.Sprintf("no stage found in workflow task: %s,taskID: %d", workflowTask.WorkflowName, workflowTask.TaskID) + logger.Error(errMsg) + return e.ErrCreateTask.AddDesc(errMsg) + } + for _, stage := range workflowTask.Stages { + if len(stage.Jobs) <= 0 { + errMsg := fmt.Sprintf("no job found in workflow task: %s,taskID: %d,stage: %s", workflowTask.WorkflowName, workflowTask.TaskID, stage.Name) + logger.Error(errMsg) + return e.ErrCreateTask.AddDesc(errMsg) + } + } + return nil +} diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/workflow_v3.go b/pkg/microservice/aslan/core/workflow/service/workflow/workflow_v3.go new file mode 100644 index 0000000000000000000000000000000000000000..aaa91ecd927693315168a14dca777a5c358977a6 --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/service/workflow/workflow_v3.go @@ -0,0 +1,286 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package workflow + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + "time" + + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" + "github.com/koderover/zadig/pkg/setting" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/log" +) + +func CreateWorkflowV3(user string, workflowModel *commonmodels.WorkflowV3, logger *zap.SugaredLogger) (string, error) { + if !checkWorkflowSubModules(workflowModel) { + errStr := "Workflow has no sub-module, please set the sub-module first" + return "", e.ErrUpsertWorkflow.AddDesc(errStr) + } + if err := ensureWorkflowV3(workflowModel, logger); err != nil { + return "", e.ErrUpsertWorkflow.AddDesc(err.Error()) + } + + workflowModel.CreatedBy = user + workflowModel.UpdatedBy = user + workflowModel.CreateTime = time.Now().Unix() + workflowModel.UpdateTime = time.Now().Unix() + + workflowID, err := commonrepo.NewWorkflowV3Coll().Create(workflowModel) + if err != nil { + logger.Errorf("Failed to create workflow v3, the error is: %s", err) + return "", e.ErrUpsertWorkflow.AddErr(err) + } + return workflowID, nil +} + +func ensureWorkflowV3(args *commonmodels.WorkflowV3, log *zap.SugaredLogger) error { + if !defaultNameRegex.MatchString(args.Name) { + log.Errorf("workflow name must match %s", defaultNameRegexString) + return fmt.Errorf("%s %s", e.InvalidFormatErrMsg, defaultNameRegexString) + } + + if err := validateV3SubTaskSetting(args.Name, args.SubTasks); err != nil { + log.Errorf("validateV3SubTaskSetting: %+v", err) + return err + } + + if workflowV3, err := commonrepo.NewWorkflowV3Coll().Find(args.Name); err == nil { + errStr := fmt.Sprintf("workflow [%s] 在项目 [%s] 中已经存在!", workflowV3.Name, workflowV3.ProjectName) + return e.ErrCreatePipeline.AddDesc(errStr) + } + + if workflow, err := commonrepo.NewWorkflowColl().Find(args.Name); err == nil { + errStr := fmt.Sprintf("workflow [%s] 在项目 [%s] 中已经存在!", workflow.Name, workflow.ProductTmplName) + return e.ErrCreatePipeline.AddDesc(errStr) + } + return nil +} + +// validateV3SubTaskSetting Validating subtasks +func validateV3SubTaskSetting(workflowName string, subtasks []map[string]interface{}) error { + for i, subTask := range subtasks { + pre, err := base.ToPreview(subTask) + if err != nil { + return errors.New(e.InterfaceToTaskErrMsg) + } + + if !pre.Enabled { + continue + } + + switch pre.TaskType { + case config.TaskBuildV3: + t, err := base.ToBuildTask(subTask) + if err != nil { + log.Error(err) + return err + } + // 设置默认 build OS 为 Ubuntu 16.04 + if t.BuildOS == "" { + t.BuildOS = setting.UbuntuXenial + t.ImageFrom = commonmodels.ImageFromKoderover + } + + ensureTaskSecretEnvs(workflowName, config.TaskBuildV3, t.JobCtx.EnvVars) + + subtasks[i], err = t.ToSubTask() + if err != nil { + return err + } + } + } + return nil +} + +func checkWorkflowSubModules(args *commonmodels.WorkflowV3) bool { + if args.SubTasks != nil && len(args.SubTasks) > 0 { + return true + } + + return false +} + +func ListWorkflowsV3(projectName string, pageNum, pageSize int64, logger *zap.SugaredLogger) ([]*WorkflowV3Brief, int64, error) { + resp := make([]*WorkflowV3Brief, 0) + workflowV3List, total, err := commonrepo.NewWorkflowV3Coll().List(&commonrepo.ListWorkflowV3Option{ + ProjectName: projectName, + }, pageNum, pageSize) + if err != nil { + logger.Errorf("Failed to list workflow v3, the error is: %s", err) + return nil, 0, err + } + for _, workflow := range workflowV3List { + resp = append(resp, &WorkflowV3Brief{ + ID: workflow.ID.Hex(), + Name: workflow.Name, + ProjectName: workflow.ProjectName, + }) + } + return resp, total, nil +} + +func GetWorkflowV3Detail(id string, logger *zap.SugaredLogger) (*WorkflowV3, error) { + resp := new(WorkflowV3) + workflow, err := commonrepo.NewWorkflowV3Coll().GetByID(id) + if err != nil { + logger.Errorf("Failed to get workflowV3 detail from id: %s, the error is: %s", id, err) + return nil, err + } + out, err := json.Marshal(workflow) + if err != nil { + logger.Errorf("Failed to unmarshal given workflow, the error is: %s", err) + return nil, err + } + err = json.Unmarshal(out, &resp) + if err != nil { + logger.Errorf("Cannot convert workflow into database model, the error is: %s", err) + return nil, err + } + return resp, nil +} + +func UpdateWorkflowV3(id, user string, workflowModel *commonmodels.WorkflowV3, logger *zap.SugaredLogger) error { + if !checkWorkflowSubModules(workflowModel) { + errStr := "Workflow has no sub-module, please set the sub-module first" + return e.ErrCreatePipeline.AddDesc(errStr) + } + workflowModel.UpdatedBy = user + workflowModel.UpdateTime = time.Now().Unix() + err := commonrepo.NewWorkflowV3Coll().Update( + id, + workflowModel, + ) + if err != nil { + logger.Errorf("update workflowV3 error: %s", err) + return e.ErrUpsertWorkflow.AddErr(err) + } + return nil +} + +func DeleteWorkflowV3(id string, logger *zap.SugaredLogger) error { + err := commonrepo.NewWorkflowV3Coll().DeleteByID(id) + if err != nil { + logger.Errorf("Failed to WorkflowV3 of id: %s, the error is: %s", id, err) + return e.ErrDeleteWorkflow.AddErr(err) + } + return nil +} + +func GetWorkflowV3Args(id string, logger *zap.SugaredLogger) ([]*WorkflowV3TaskArgs, error) { + workflow, err := commonrepo.NewWorkflowV3Coll().GetByID(id) + if err != nil { + logger.Errorf("Failed to get workflowV3 detail from id: %s, the error is: %s", id, err) + return nil, err + } + resp := make([]*WorkflowV3TaskArgs, 0) + for _, param := range workflow.Parameters { + switch param.Type { + case "string": + resp = append(resp, &WorkflowV3TaskArgs{ + Type: string(StringType), + Key: param.Key, + Value: param.DefaultValue, + }) + case "choice": + resp = append(resp, &WorkflowV3TaskArgs{ + Type: string(ChoiceType), + Key: param.Key, + Value: param.DefaultValue, + Choice: param.ChoiceOption, + }) + case "external": + externalEnv := &WorkflowV3TaskArgs{ + Type: string(ExternalType), + } + for _, kv := range param.ExternalSetting.Params { + if kv.Display { + externalEnv.Key = kv.ParamKey + break + } + } + if externalEnv.Key == "" { + errorMsg := fmt.Sprintf("error getting external key, cannot find the display key") + logger.Error(errorMsg) + return nil, errors.New(errorMsg) + } + options, err := getEnvsFromExternalSystem(param.ExternalSetting, logger) + if err != nil { + logger.Errorf("Failed to get response from external system, the error is: %s", err) + return nil, err + } + externalEnv.Options = options + resp = append(resp, externalEnv) + default: + return nil, fmt.Errorf("parameter of type %s is not supported", param.Type) + } + } + return resp, nil +} + +func getEnvsFromExternalSystem(setting *commonmodels.ExternalSetting, logger *zap.SugaredLogger) ([]map[string]interface{}, error) { + externalSystem, err := commonrepo.NewExternalSystemColl().GetByID(setting.SystemID) + if err != nil { + return nil, err + } + client := http.Client{} + requestPath := fmt.Sprintf("%s/%s", externalSystem.Server, setting.Endpoint) + req, err := http.NewRequest(setting.Method, requestPath, strings.NewReader(setting.Body)) + if err != nil { + return nil, err + } + if setting.Headers != nil { + for _, header := range setting.Headers { + req.Header.Set(header.Key, header.Value) + } + } + resp, err := client.Do(req) + if err != nil { + logger.Errorf("failed to get env from external system, the error is: %s", err) + return nil, err + } + if resp.StatusCode != 200 { + logger.Errorf("failed to get env from external system, the status code is: %d", resp.StatusCode) + errMsg := fmt.Sprintf("got response code %d from external system", resp.StatusCode) + return nil, errors.New(errMsg) + } + decoder := json.NewDecoder(resp.Body) + respList := make([]map[string]interface{}, 0) + err = decoder.Decode(&respList) + if err != nil { + return nil, err + } + envList := make([]map[string]interface{}, 0) + // find every single key required + for _, respItem := range respList { + item := map[string]interface{}{} + for _, kv := range setting.Params { + item[kv.ParamKey] = respItem[kv.ResponseKey] + } + envList = append(envList, item) + } + return envList, nil +} diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/workflow_v4.go b/pkg/microservice/aslan/core/workflow/service/workflow/workflow_v4.go new file mode 100644 index 0000000000000000000000000000000000000000..6819ed671ab2c4b02783563f25e4fcf04a23156d --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/service/workflow/workflow_v4.go @@ -0,0 +1,389 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package workflow + +import ( + "fmt" + "regexp" + "time" + + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" + commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + jobctl "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow/job" + "github.com/koderover/zadig/pkg/setting" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/log" +) + +const ( + JobNameRegx = "^[a-z][a-z0-9-]{0,31}$" + WorkflowRegx = "^[a-z0-9-]{1,32}$" +) + +func CreateWorkflowV4(user string, workflow *commonmodels.WorkflowV4, logger *zap.SugaredLogger) error { + existedWorkflow, err := commonrepo.NewWorkflowV4Coll().Find(workflow.Name) + if err == nil { + errStr := fmt.Sprintf("workflow v4 [%s] 在项目 [%s] 中已经存在!", workflow.Name, existedWorkflow.Project) + return e.ErrUpsertWorkflow.AddDesc(errStr) + } + if err := LintWorkflowV4(workflow, logger); err != nil { + return err + } + + workflow.CreatedBy = user + workflow.UpdatedBy = user + workflow.CreateTime = time.Now().Unix() + workflow.UpdateTime = time.Now().Unix() + + for _, stage := range workflow.Stages { + for _, job := range stage.Jobs { + if err := jobctl.Instantiate(job, workflow); err != nil { + logger.Errorf("Failed to instantiate workflow v4,error: %v", err) + return e.ErrUpsertWorkflow.AddErr(err) + } + } + } + + if _, err := commonrepo.NewWorkflowV4Coll().Create(workflow); err != nil { + logger.Errorf("Failed to create workflow v4, the error is: %s", err) + return e.ErrUpsertWorkflow.AddErr(err) + } + + return nil +} + +func UpdateWorkflowV4(name, user string, inputWorkflow *commonmodels.WorkflowV4, logger *zap.SugaredLogger) error { + workflow, err := commonrepo.NewWorkflowV4Coll().Find(name) + if err != nil { + logger.Errorf("Failed to find WorkflowV4: %s, the error is: %v", name, err) + return e.ErrFindWorkflow.AddErr(err) + } + if err := LintWorkflowV4(inputWorkflow, logger); err != nil { + return err + } + + inputWorkflow.UpdatedBy = user + inputWorkflow.UpdateTime = time.Now().Unix() + inputWorkflow.ID = workflow.ID + + for _, stage := range inputWorkflow.Stages { + for _, job := range stage.Jobs { + if err := jobctl.Instantiate(job, workflow); err != nil { + logger.Errorf("Failed to instantiate workflow v4,error: %v", err) + return e.ErrUpsertWorkflow.AddErr(err) + } + } + } + + if err := commonrepo.NewWorkflowV4Coll().Update( + workflow.ID.Hex(), + inputWorkflow, + ); err != nil { + logger.Errorf("update workflowV4 error: %s", err) + return e.ErrUpsertWorkflow.AddErr(err) + } + return nil +} + +func FindWorkflowV4(encryptedKey, name string, logger *zap.SugaredLogger) (*commonmodels.WorkflowV4, error) { + workflow, err := commonrepo.NewWorkflowV4Coll().Find(name) + if err != nil { + logger.Errorf("Failed to find WorkflowV4: %s, the error is: %v", name, err) + return workflow, e.ErrFindWorkflow.AddErr(err) + } + if err := ensureWorkflowV4Resp(encryptedKey, workflow, logger); err != nil { + return workflow, err + } + return workflow, err +} + +func DeleteWorkflowV4(name string, logger *zap.SugaredLogger) error { + workflow, err := commonrepo.NewWorkflowV4Coll().Find(name) + if err != nil { + logger.Errorf("Failed to delete WorkflowV4: %s, the error is: %v", name, err) + return e.ErrDeleteWorkflow.AddErr(err) + } + if err := commonrepo.NewWorkflowV4Coll().DeleteByID(workflow.ID.Hex()); err != nil { + logger.Errorf("Failed to delete WorkflowV4: %s, the error is: %v", name, err) + return e.ErrDeleteWorkflow.AddErr(err) + } + if err := commonrepo.NewworkflowTaskv4Coll().DeleteByWorkflowName(name); err != nil { + logger.Errorf("Failed to delete WorkflowV4 task: %s, the error is: %v", name, err) + return e.ErrDeleteWorkflow.AddErr(err) + } + if err := commonrepo.NewCounterColl().Delete("WorkflowTaskV4:" + name); err != nil { + log.Errorf("Counter.Delete error: %s", err) + } + return nil +} + +func ListWorkflowV4(projectName, userID string, names []string, ignoreWorkflow bool, logger *zap.SugaredLogger) ([]*Workflow, error) { + resp := make([]*Workflow, 0) + workflowV4List, _, err := commonrepo.NewWorkflowV4Coll().List(&commonrepo.ListWorkflowV4Option{ + ProjectName: projectName, + }, 0, 0) + if err != nil { + logger.Errorf("Failed to list workflow v4, the error is: %s", err) + return resp, err + } + + workflow := []*Workflow{} + + if !ignoreWorkflow { + workflow, err = ListWorkflows([]string{projectName}, userID, names, logger) + if err != nil { + return resp, err + } + } + + workflowList := []string{} + for _, wV4 := range workflowV4List { + workflowList = append(workflowList, wV4.Name) + } + resp = append(resp, workflow...) + tasks, _, err := commonrepo.NewworkflowTaskv4Coll().List(&commonrepo.ListWorkflowTaskV4Option{WorkflowNames: workflowList}) + if err != nil { + return resp, err + } + + for _, workflowModel := range workflowV4List { + stages := []string{} + for _, stage := range workflowModel.Stages { + stages = append(stages, stage.Name) + } + workflow := &Workflow{ + Name: workflowModel.Name, + ProjectName: workflowModel.Project, + EnabledStages: stages, + CreateTime: workflowModel.CreateTime, + UpdateTime: workflowModel.UpdateTime, + UpdateBy: workflowModel.UpdatedBy, + WorkflowType: "common_workflow", + Description: workflowModel.Description, + } + getRecentTaskV4Info(workflow, tasks) + + resp = append(resp, workflow) + } + return resp, nil +} + +func getRecentTaskV4Info(workflow *Workflow, tasks []*commonmodels.WorkflowTask) { + recentTask := &commonmodels.WorkflowTask{} + recentFailedTask := &commonmodels.WorkflowTask{} + recentSucceedTask := &commonmodels.WorkflowTask{} + for _, task := range tasks { + if task.WorkflowName != workflow.Name { + continue + } + if task.TaskID > recentTask.TaskID { + recentTask = task + } + if task.Status == config.StatusPassed && task.TaskID > recentSucceedTask.TaskID { + recentSucceedTask = task + } + if task.Status == config.StatusFailed && task.TaskID > recentFailedTask.TaskID { + recentFailedTask = task + } + } + if recentTask.TaskID > 0 { + workflow.RecentTask = &TaskInfo{ + TaskID: recentTask.TaskID, + PipelineName: recentTask.WorkflowName, + Status: string(recentTask.Status), + } + } + if recentSucceedTask.TaskID > 0 { + workflow.RecentSuccessfulTask = &TaskInfo{ + TaskID: recentSucceedTask.TaskID, + PipelineName: recentSucceedTask.WorkflowName, + Status: string(recentSucceedTask.Status), + } + } + if recentFailedTask.TaskID > 0 { + workflow.RecentFailedTask = &TaskInfo{ + TaskID: recentFailedTask.TaskID, + PipelineName: recentFailedTask.WorkflowName, + Status: string(recentFailedTask.Status), + } + } +} + +func ensureWorkflowV4Resp(encryptedKey string, workflow *commonmodels.WorkflowV4, logger *zap.SugaredLogger) error { + for _, stage := range workflow.Stages { + for _, job := range stage.Jobs { + if job.JobType == config.JobZadigBuild { + spec := &commonmodels.ZadigBuildJobSpec{} + if err := commonmodels.IToi(job.Spec, spec); err != nil { + logger.Errorf(err.Error()) + return e.ErrFindWorkflow.AddErr(err) + } + for _, build := range spec.ServiceAndBuilds { + buildInfo, err := commonrepo.NewBuildColl().Find(&commonrepo.BuildFindOption{Name: build.BuildName}) + if err != nil { + logger.Errorf(err.Error()) + return e.ErrFindWorkflow.AddErr(err) + } + kvs := buildInfo.PreBuild.Envs + if buildInfo.TemplateID != "" { + templateEnvs := []*commonmodels.KeyVal{} + buildTemplate, err := commonrepo.NewBuildTemplateColl().Find(&commonrepo.BuildTemplateQueryOption{ + ID: buildInfo.TemplateID, + }) + // if template not found, envs are empty, but do not block user. + if err != nil { + logger.Error("build job: %s, template not found", buildInfo.Name) + } else { + templateEnvs = buildTemplate.PreBuild.Envs + } + + for _, target := range buildInfo.Targets { + if target.ServiceName == build.ServiceName && target.ServiceModule == build.ServiceModule { + kvs = target.Envs + } + } + // if build template update any keyvals, merge it. + kvs = commonservice.MergeBuildEnvs(templateEnvs, kvs) + } + build.KeyVals = commonservice.MergeBuildEnvs(kvs, build.KeyVals) + if err := commonservice.EncryptKeyVals(encryptedKey, build.KeyVals, logger); err != nil { + logger.Errorf(err.Error()) + return e.ErrFindWorkflow.AddErr(err) + } + } + job.Spec = spec + } + if job.JobType == config.JobFreestyle { + spec := &commonmodels.FreestyleJobSpec{} + if err := commonmodels.IToi(job.Spec, spec); err != nil { + logger.Errorf(err.Error()) + return e.ErrFindWorkflow.AddErr(err) + } + if err := commonservice.EncryptKeyVals(encryptedKey, spec.Properties.Envs, logger); err != nil { + logger.Errorf(err.Error()) + return e.ErrFindWorkflow.AddErr(err) + } + job.Spec = spec + } + if job.JobType == config.JobPlugin { + spec := &commonmodels.PluginJobSpec{} + if err := commonmodels.IToi(job.Spec, spec); err != nil { + logger.Errorf(err.Error()) + return e.ErrFindWorkflow.AddErr(err) + } + if err := commonservice.EncryptParams(encryptedKey, spec.Plugin.Inputs, logger); err != nil { + logger.Errorf(err.Error()) + return e.ErrFindWorkflow.AddErr(err) + } + job.Spec = spec + } + } + } + return nil +} + +func LintWorkflowV4(workflow *commonmodels.WorkflowV4, logger *zap.SugaredLogger) error { + if workflow.Project == "" { + err := fmt.Errorf("project should not be empty") + logger.Errorf(err.Error()) + return e.ErrUpsertWorkflow.AddErr(err) + } + match, err := regexp.MatchString(WorkflowRegx, workflow.Name) + if err != nil { + logger.Errorf("reg compile failed: %v", err) + return e.ErrUpsertWorkflow.AddErr(err) + } + if !match { + err := fmt.Errorf("workflow name should match %s", WorkflowRegx) + logger.Errorf(err.Error()) + return e.ErrUpsertWorkflow.AddErr(err) + } + project, err := templaterepo.NewProductColl().Find(workflow.Project) + if err != nil { + logger.Errorf("Failed to get project %s, error: %v", workflow.Project, err) + return e.ErrUpsertWorkflow.AddErr(err) + } + + if project.ProductFeature != nil { + if project.ProductFeature.DeployType != setting.K8SDeployType && project.ProductFeature.DeployType != setting.HelmDeployType { + logger.Error("common workflow only support k8s and helm project") + return e.ErrUpsertWorkflow.AddDesc("common workflow only support k8s and helm project") + } + } + stageNameMap := make(map[string]bool) + jobNameMap := make(map[string]string) + // deploy job can not qoute a build job which runs after it. + buildJobNameMap := make(map[string]string) + + reg, err := regexp.Compile(JobNameRegx) + if err != nil { + logger.Errorf("reg compile failed: %v", err) + return e.ErrUpsertWorkflow.AddErr(err) + } + for _, stage := range workflow.Stages { + if _, ok := stageNameMap[stage.Name]; !ok { + stageNameMap[stage.Name] = true + } else { + logger.Errorf("duplicated stage name: %s", stage.Name) + return e.ErrUpsertWorkflow.AddDesc(fmt.Sprintf("duplicated job name: %s", stage.Name)) + } + stageBuildJobNameMap := make(map[string]string) + for _, job := range stage.Jobs { + if !stage.Parallel { + buildJobNameMap[job.Name] = string(job.JobType) + } else { + stageBuildJobNameMap[job.Name] = string(job.JobType) + } + if match := reg.MatchString(job.Name); !match { + logger.Errorf("job name [%s] did not match %s", job.Name, JobNameRegx) + return e.ErrUpsertWorkflow.AddDesc(fmt.Sprintf("job name [%s] did not match %s", job.Name, JobNameRegx)) + } + if _, ok := jobNameMap[job.Name]; !ok { + jobNameMap[job.Name] = string(job.JobType) + } else { + logger.Errorf("duplicated job name: %s", job.Name) + return e.ErrUpsertWorkflow.AddDesc(fmt.Sprintf("duplicated job name: %s", job.Name)) + } + + if job.JobType == config.JobZadigDeploy { + spec := &commonmodels.ZadigDeployJobSpec{} + if err := commonmodels.IToiYaml(job.Spec, spec); err != nil { + logger.Errorf("decode job spec error: %v", err) + return e.ErrUpsertWorkflow.AddErr(err) + } + if spec.Source != config.SourceFromJob { + continue + } + jobType, ok := buildJobNameMap[spec.JobName] + if !ok || jobType != string(config.JobZadigBuild) { + errMsg := fmt.Sprintf("can not quote job %s in job %s", spec.JobName, job.Name) + logger.Error(errMsg) + return e.ErrUpsertWorkflow.AddDesc(errMsg) + } + } + } + for k, v := range stageBuildJobNameMap { + buildJobNameMap[k] = v + } + } + return nil +} diff --git a/pkg/microservice/aslan/core/workflow/testing/handler/router.go b/pkg/microservice/aslan/core/workflow/testing/handler/router.go index e0701d22cd020586fa6cf90fe5f58babbf3f9d8f..b01b4f345913aed3497991514bad9a28e8e46249 100644 --- a/pkg/microservice/aslan/core/workflow/testing/handler/router.go +++ b/pkg/microservice/aslan/core/workflow/testing/handler/router.go @@ -18,9 +18,6 @@ package handler import ( "github.com/gin-gonic/gin" - - gin2 "github.com/koderover/zadig/pkg/middleware/gin" - "github.com/koderover/zadig/pkg/types/permission" ) type Router struct{} @@ -32,8 +29,6 @@ func (*Router) Inject(router *gin.RouterGroup) { testReport.GET("", GetHTMLTestReport) } - router.Use(gin2.Auth()) - // --------------------------------------------------------------------------------------- // 系统测试接口 // --------------------------------------------------------------------------------------- @@ -49,11 +44,31 @@ func (*Router) Inject(router *gin.RouterGroup) { // --------------------------------------------------------------------------------------- tester := router.Group("test") { - tester.POST("", GetTestProductName, gin2.IsHavePermission([]string{permission.TestManageUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, CreateTestModule) - tester.PUT("", GetTestProductName, gin2.IsHavePermission([]string{permission.TestManageUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, UpdateTestModule) + tester.POST("", GetTestProductName, CreateTestModule) + tester.PUT("", GetTestProductName, UpdateTestModule) tester.GET("", ListTestModules) tester.GET("/:name", GetTestModule) - tester.DELETE("/:name", gin2.IsHavePermission([]string{permission.TestDeleteUUID}, permission.QueryType), gin2.UpdateOperationLogStatus, DeleteTestModule) + tester.DELETE("/:name", DeleteTestModule) + } + + // --------------------------------------------------------------------------------------- + // Code scan APIs + // --------------------------------------------------------------------------------------- + scanner := router.Group("scanning") + { + // code scan config apis + scanner.POST("", GetScanningProductName, CreateScanningModule) + scanner.PUT("/:id", GetScanningProductName, UpdateScanningModule) + scanner.GET("", ListScanningModule) + scanner.GET("/:id", GetScanningModule) + scanner.DELETE("/:id", DeleteScanningModule) + + // code scan tasks apis + scanner.POST("/:id/task", CreateScanningTask) + scanner.GET("/:id/task", ListScanningTask) + scanner.GET("/:id/task/:scan_id", GetScanningTask) + scanner.DELETE("/:id/task/:scan_id", CancelScanningTask) + scanner.GET("/:id/task/:scan_id/sse", GetScanningTaskSSE) } testStat := router.Group("teststat") @@ -72,9 +87,9 @@ func (*Router) Inject(router *gin.RouterGroup) { // --------------------------------------------------------------------------------------- testTask := router.Group("testtask") { - testTask.POST("", gin2.UpdateOperationLogStatus, CreateTestTask) - testTask.POST("/productName/:productName/id/:id/pipelines/:name/restart", gin2.UpdateOperationLogStatus, RestartTestTask) - testTask.DELETE("/productName/:productName/id/:id/pipelines/:name", gin2.UpdateOperationLogStatus, CancelTestTaskV2) + testTask.POST("", CreateTestTask) + testTask.POST("/productName/:productName/id/:id/pipelines/:name/restart", RestartTestTask) + testTask.DELETE("/productName/:productName/id/:id/pipelines/:name", CancelTestTaskV2) } // --------------------------------------------------------------------------------------- diff --git a/pkg/microservice/aslan/core/workflow/testing/handler/scanning.go b/pkg/microservice/aslan/core/workflow/testing/handler/scanning.go new file mode 100644 index 0000000000000000000000000000000000000000..59afff728816bd9faef8d2872c9002227690d02d --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/testing/handler/scanning.go @@ -0,0 +1,295 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + e "github.com/koderover/zadig/pkg/tool/errors" + "io/ioutil" + "k8s.io/apimachinery/pkg/util/wait" + "strconv" + "time" + + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/testing/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + "github.com/koderover/zadig/pkg/tool/log" +) + +var MissingIDError = e.ErrInvalidParam.AddDesc("ID must be provided") + +func GetScanningProductName(c *gin.Context) { + args := new(commonmodels.Scanning) + data, err := c.GetRawData() + if err != nil { + log.Errorf("c.GetRawData() err : %s", err) + return + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("json.Unmarshal err : %s", err) + return + } + c.Set("productName", args.ProjectName) + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + c.Next() +} + +func CreateScanningModule(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(service.Scanning) + data, err := c.GetRawData() + if err != nil { + log.Errorf("Create scanning module c.GetRawData() err : %s", err) + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("Create scanning module json.Unmarshal err : %s", err) + } + + internalhandler.InsertOperationLog(c, ctx.UserName, args.ProjectName, "新增", "项目管理-代码扫描", args.Name, string(data), ctx.Logger) + + if err != nil { + ctx.Err = fmt.Errorf("create scanning module err : %s", err) + return + } + + ctx.Err = service.CreateScanningModule(ctx.UserName, args, ctx.Logger) +} + +func UpdateScanningModule(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(service.Scanning) + data, err := c.GetRawData() + if err != nil { + log.Errorf("Update scanning module c.GetRawData() err : %s", err) + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("Update scanning module json.Unmarshal err : %s", err) + } + + internalhandler.InsertOperationLog(c, ctx.UserName, args.ProjectName, "修改", "项目管理-代码扫描", args.Name, string(data), ctx.Logger) + + if err != nil { + ctx.Err = fmt.Errorf("update scanning module err : %s", err) + return + } + + id := c.Param("id") + if id == "" { + ctx.Err = MissingIDError + return + } + + ctx.Err = service.UpdateScanningModule(id, ctx.UserName, args, ctx.Logger) +} + +func ListScanningModule(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + resp, _, err := service.ListScanningModule(c.Query("projectName"), ctx.Logger) + ctx.Resp = resp + ctx.Err = err +} + +func GetScanningModule(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + id := c.Param("id") + if id == "" { + ctx.Err = MissingIDError + return + } + + ctx.Resp, ctx.Err = service.GetScanningModuleByID(id, ctx.Logger) +} + +func DeleteScanningModule(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + internalhandler.InsertOperationLog(c, ctx.UserName, c.Query("projectName"), "删除", "项目管理-测试", c.Param("name"), "", ctx.Logger) + + id := c.Param("id") + if id == "" { + ctx.Err = MissingIDError + return + } + + ctx.Err = service.DeleteScanningModuleByID(id, ctx.Logger) +} + +type createScanningTaskResp struct { + TaskID int64 `json:"task_id"` +} + +func CreateScanningTask(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + id := c.Param("id") + if id == "" { + ctx.Err = MissingIDError + return + } + + req := make([]*service.ScanningRepoInfo, 0) + data, err := c.GetRawData() + if err != nil { + log.Errorf("Create scanning task c.GetRawData() err : %v", err) + } + if err = json.Unmarshal(data, &req); err != nil { + log.Errorf("Create scanning task json.Unmarshal err : %v", err) + } + + internalhandler.InsertOperationLog(c, ctx.UserName, "", "新增", "代码扫描任务", id, string(data), ctx.Logger) + + resp, err := service.CreateScanningTask(id, req, "", ctx.UserName, ctx.Logger) + if err != nil { + ctx.Err = err + return + } + ctx.Resp = &createScanningTaskResp{TaskID: resp} +} + +type listQuery struct { + PageSize int64 `json:"page_size" form:"page_size,default=100"` + PageNum int64 `json:"page_num" form:"page_num,default=1"` +} + +func ListScanningTask(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + id := c.Param("id") + if id == "" { + ctx.Err = MissingIDError + return + } + + // Query Verification + args := &listQuery{} + if err := c.ShouldBindQuery(args); err != nil { + ctx.Err = err + return + } + + ctx.Resp, ctx.Err = service.ListScanningTask(id, args.PageNum, args.PageSize, ctx.Logger) +} + +func GetScanningTask(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + id := c.Param("id") + if id == "" { + ctx.Err = MissingIDError + return + } + + taskIDStr := c.Param("scan_id") + if taskIDStr == "" { + ctx.Err = fmt.Errorf("scan_id must be provided") + return + } + + taskID, err := strconv.ParseInt(taskIDStr, 10, 64) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc(fmt.Sprintf("invalid task id: %s", err)) + return + } + + ctx.Resp, ctx.Err = service.GetScanningTaskInfo(id, taskID, ctx.Logger) +} + +func CancelScanningTask(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + id := c.Param("id") + if id == "" { + ctx.Err = MissingIDError + return + } + + taskIDStr := c.Param("scan_id") + if taskIDStr == "" { + ctx.Err = fmt.Errorf("scan_id must be provided") + return + } + + taskID, err := strconv.ParseInt(taskIDStr, 10, 64) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc(fmt.Sprintf("invalid task id: %s", err)) + return + } + + internalhandler.InsertOperationLog(c, ctx.UserName, "", "取消", "代码扫描任务", id, taskIDStr, ctx.Logger) + + ctx.Err = service.CancelScanningTask(ctx.UserName, id, taskID, config.ScanningType, ctx.RequestID, ctx.Logger) +} + +func GetScanningTaskSSE(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + id := c.Param("id") + if id == "" { + ctx.Err = MissingIDError + return + } + + taskIDStr := c.Param("scan_id") + if taskIDStr == "" { + ctx.Err = fmt.Errorf("scan_id must be provided") + return + } + + taskID, err := strconv.ParseInt(taskIDStr, 10, 64) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc(fmt.Sprintf("invalid task id: %s", err)) + return + } + + internalhandler.Stream(c, func(ctx1 context.Context, msgChan chan interface{}) { + err := wait.PollImmediateUntil(time.Second, func() (bool, error) { + res, err := service.GetScanningTaskInfo(id, taskID, ctx.Logger) + if err != nil { + ctx.Logger.Errorf("[%s] Get scanning task info error: %s", ctx.UserName, err) + return false, err + } + + msgChan <- res + + return false, nil + }, ctx1.Done()) + + if err != nil && err != wait.ErrWaitTimeout { + ctx.Logger.Error(err) + } + }, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/workflow/testing/handler/test_detail.go b/pkg/microservice/aslan/core/workflow/testing/handler/test_detail.go index 02295c3e80f3a4325961e0c5d696fff4fe99d7eb..492db2a97cff71a4f51ec4163ba0f6f8a37fa60f 100644 --- a/pkg/microservice/aslan/core/workflow/testing/handler/test_detail.go +++ b/pkg/microservice/aslan/core/workflow/testing/handler/test_detail.go @@ -27,5 +27,5 @@ func ListDetailTestModules(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = service.ListTestingDetails(c.Query("productName"), ctx.Logger) + ctx.Resp, ctx.Err = service.ListTestingDetails(c.Query("projectName"), ctx.Logger) } diff --git a/pkg/microservice/aslan/core/workflow/testing/handler/test_task.go b/pkg/microservice/aslan/core/workflow/testing/handler/test_task.go index 80ebf4911102eb5698b306f3889f6dd2e62b938f..ae1544284962104463b214e7ccd3507ec00acbbb 100644 --- a/pkg/microservice/aslan/core/workflow/testing/handler/test_task.go +++ b/pkg/microservice/aslan/core/workflow/testing/handler/test_task.go @@ -49,7 +49,7 @@ func CreateTestTask(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("CreateTestTask json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, args.ProductName, "新增", "测试-task", fmt.Sprintf("%s-%s", args.TestName, "job"), string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, args.ProductName, "新增", "测试-task", fmt.Sprintf("%s-%s", args.TestName, "job"), string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) if err := c.ShouldBindWith(&args, binding.JSON); err != nil { @@ -58,13 +58,12 @@ func CreateTestTask(c *gin.Context) { } if args.TestTaskCreator != setting.CronTaskCreator && args.TestTaskCreator != setting.WebhookTaskCreator { - args.TestTaskCreator = ctx.Username + args.TestTaskCreator = ctx.UserName } ctx.Resp, ctx.Err = service.CreateTestTask(args, ctx.Logger) - // 发送通知 if ctx.Err != nil { - commonservice.SendFailedTaskMessage(ctx.Username, args.ProductName, args.TestName, ctx.RequestID, config.TestType, ctx.Err, ctx.Logger) + commonservice.SendFailedTaskMessage(ctx.UserName, args.ProductName, args.TestName, ctx.RequestID, config.TestType, ctx.Err, ctx.Logger) } } @@ -72,7 +71,7 @@ func RestartTestTask(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - internalhandler.InsertOperationLog(c, ctx.Username, c.Param("productName"), "重启", "测试任务", c.Param("name"), "", ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, c.Param("productName"), "重启", "测试任务", c.Param("name"), "", ctx.Logger) taskID, err := strconv.ParseInt(c.Param("id"), 10, 64) if err != nil { @@ -80,18 +79,18 @@ func RestartTestTask(c *gin.Context) { return } - ctx.Err = workflowservice.RestartPipelineTaskV2(ctx.Username, taskID, c.Param("name"), config.TestType, ctx.Logger) + ctx.Err = workflowservice.RestartPipelineTaskV2(ctx.UserName, taskID, c.Param("name"), config.TestType, ctx.Logger) } func CancelTestTaskV2(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - internalhandler.InsertOperationLog(c, ctx.Username, c.Param("productName"), "取消", "测试任务", c.Param("name"), "", ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, c.Param("productName"), "取消", "测试任务", c.Param("name"), "", ctx.Logger) taskID, err := strconv.ParseInt(c.Param("id"), 10, 64) if err != nil { ctx.Err = e.ErrInvalidParam.AddDesc("invalid task id") return } - ctx.Err = commonservice.CancelTaskV2(ctx.Username, c.Param("name"), taskID, config.TestType, ctx.RequestID, ctx.Logger) + ctx.Err = commonservice.CancelTaskV2(ctx.UserName, c.Param("name"), taskID, config.TestType, ctx.RequestID, ctx.Logger) } diff --git a/pkg/microservice/aslan/core/workflow/testing/handler/testing.go b/pkg/microservice/aslan/core/workflow/testing/handler/testing.go index 24c35774d495fbe51269fae99b975e3a29724f10..d7df15dd9d94c4f7addbc9c08ed0492884ff1561 100644 --- a/pkg/microservice/aslan/core/workflow/testing/handler/testing.go +++ b/pkg/microservice/aslan/core/workflow/testing/handler/testing.go @@ -24,6 +24,7 @@ import ( "github.com/gin-gonic/gin" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/testing/service" internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" @@ -59,7 +60,7 @@ func CreateTestModule(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("CreateTestModule json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, args.ProductName, "新增", "项目管理-测试", args.Name, string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, args.ProductName, "新增", "项目管理-测试", args.Name, string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) err = c.BindJSON(args) @@ -68,7 +69,7 @@ func CreateTestModule(c *gin.Context) { return } - ctx.Err = service.CreateTesting(ctx.Username, args, ctx.Logger) + ctx.Err = service.CreateTesting(ctx.UserName, args, ctx.Logger) } func UpdateTestModule(c *gin.Context) { @@ -83,7 +84,7 @@ func UpdateTestModule(c *gin.Context) { if err = json.Unmarshal(data, args); err != nil { log.Errorf("UpdateTestModule json.Unmarshal err : %v", err) } - internalhandler.InsertOperationLog(c, ctx.Username, args.ProductName, "更新", "项目管理-测试", args.Name, string(data), ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, args.ProductName, "更新", "项目管理-测试", args.Name, string(data), ctx.Logger) c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) err = c.BindJSON(args) @@ -92,14 +93,14 @@ func UpdateTestModule(c *gin.Context) { return } - ctx.Err = service.UpdateTesting(ctx.Username, args, ctx.Logger) + ctx.Err = service.UpdateTesting(ctx.UserName, args, ctx.Logger) } func ListTestModules(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = service.ListTestingOpt(c.Query("productName"), c.Query("testType"), ctx.Logger) + ctx.Resp, ctx.Err = service.ListTestingOpt(c.Query("projectName"), c.Query("testType"), ctx.Logger) } func GetTestModule(c *gin.Context) { @@ -112,14 +113,14 @@ func GetTestModule(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("empty Name") return } - ctx.Resp, ctx.Err = service.GetTesting(name, c.Query("productName"), ctx.Logger) + ctx.Resp, ctx.Err = service.GetTesting(name, c.Query("projectName"), ctx.Logger) } func DeleteTestModule(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - internalhandler.InsertOperationLog(c, ctx.Username, c.Query("productName"), "删除", "项目管理-测试", c.Param("name"), "", ctx.Logger) + internalhandler.InsertOperationLog(c, ctx.UserName, c.Query("projectName"), "删除", "项目管理-测试", c.Param("name"), "", ctx.Logger) name := c.Param("name") if name == "" { @@ -127,7 +128,7 @@ func DeleteTestModule(c *gin.Context) { return } - ctx.Err = service.DeleteTestModule(name, c.Query("productName"), ctx.RequestID, ctx.Logger) + ctx.Err = commonservice.DeleteTestModule(name, c.Query("projectName"), ctx.RequestID, ctx.Logger) } func GetHTMLTestReport(c *gin.Context) { diff --git a/pkg/microservice/aslan/core/workflow/testing/handler/workspace.go b/pkg/microservice/aslan/core/workflow/testing/handler/workspace.go index 3c8ef86f6159fed630baeaedbe98783f30db2be2..69171697ad1c1ae44cd55afc245c00bf2f215132 100644 --- a/pkg/microservice/aslan/core/workflow/testing/handler/workspace.go +++ b/pkg/microservice/aslan/core/workflow/testing/handler/workspace.go @@ -38,5 +38,5 @@ func GetTestArtifactInfo(c *gin.Context) { dir := c.Query("dir") - _, ctx.Resp, ctx.Err = service.GetTestArtifactInfo(c.Param("pipelineName"), dir, taskID, ctx.Logger) + ctx.Resp, ctx.Err = service.GetTestArtifactInfo(c.Param("pipelineName"), dir, taskID, ctx.Logger) } diff --git a/pkg/microservice/aslan/core/workflow/testing/service/scanning.go b/pkg/microservice/aslan/core/workflow/testing/service/scanning.go new file mode 100644 index 0000000000000000000000000000000000000000..96d956cffd5c895a0c5feb5465f8edb5c94dff7c --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/testing/service/scanning.go @@ -0,0 +1,454 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "context" + "fmt" + "sort" + "strings" + "time" + + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/scmnotify" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/webhook" + workflowservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/types" +) + +func CreateScanningModule(username string, args *Scanning, log *zap.SugaredLogger) error { + if len(args.Name) == 0 { + return e.ErrCreateScanningModule.AddDesc("empty Name") + } + + err := commonservice.ProcessWebhook(args.AdvancedSetting.HookCtl.Items, nil, webhook.ScannerPrefix+args.Name, log) + if err != nil { + return e.ErrCreateScanningModule.AddErr(err) + } + + scanningModule := ConvertToDBScanningModule(args) + scanningModule.UpdatedBy = username + + err = commonrepo.NewScanningColl().Create(scanningModule) + + if err != nil { + log.Errorf("Create scanning module %s error: %s", args.Name, err) + return e.ErrCreateScanningModule.AddErr(err) + } + + return nil +} + +func UpdateScanningModule(id, username string, args *Scanning, log *zap.SugaredLogger) error { + if len(args.Name) == 0 { + return e.ErrCreateScanningModule.AddDesc("empty Name") + } + + scanning, err := commonrepo.NewScanningColl().GetByID(id) + if err != nil { + log.Errorf("failed to get scanning information to update webhook, err: %s", err) + return err + } + + if scanning.AdvancedSetting.HookCtl.Enabled { + err = commonservice.ProcessWebhook(args.AdvancedSetting.HookCtl.Items, scanning.AdvancedSetting.HookCtl.Items, webhook.ScannerPrefix+args.Name, log) + if err != nil { + log.Errorf("failed to process webhook for scanning: %s, the error is: %s", args.Name, err) + return e.ErrUpdateScanningModule.AddErr(err) + } + } else { + err = commonservice.ProcessWebhook(args.AdvancedSetting.HookCtl.Items, nil, webhook.ScannerPrefix+args.Name, log) + if err != nil { + log.Errorf("failed to process webhook for scanning: %s, the error is: %s", args.Name, err) + return e.ErrUpdateScanningModule.AddErr(err) + } + } + + scanningModule := ConvertToDBScanningModule(args) + scanningModule.UpdatedBy = username + + err = commonrepo.NewScanningColl().Update(id, scanningModule) + + if err != nil { + log.Errorf("update scanning module %s error: %s", args.Name, err) + return e.ErrUpdateScanningModule.AddErr(err) + } + + return nil +} + +func ListScanningModule(projectName string, log *zap.SugaredLogger) ([]*ListScanningRespItem, int64, error) { + scanningList, total, err := commonrepo.NewScanningColl().List(&commonrepo.ScanningListOption{ProjectName: projectName}, 0, 0) + if err != nil { + log.Errorf("failed to list scanning list from mongodb, the error is: %s", err) + return nil, 0, err + } + resp := make([]*ListScanningRespItem, 0) + for _, scanning := range scanningList { + res, err := ListScanningTask(scanning.ID.Hex(), 0, 0, log) + if err != nil { + log.Errorf("failed to get scanning task statistics, error is: %s", err) + return nil, 0, err + } + var timesTaken int64 + for _, scanTask := range res.ScanTasks { + timesTaken += scanTask.RunTime + } + var avgRuntime int64 + if len(res.ScanTasks) > 0 { + avgRuntime = timesTaken / int64(len(res.ScanTasks)) + } else { + avgRuntime = 0 + } + resp = append(resp, &ListScanningRespItem{ + ID: scanning.ID.Hex(), + Name: scanning.Name, + Statistics: &ScanningStatistic{ + TimesRun: int64(res.TotalTasks), + AverageRuntime: avgRuntime, + }, + CreatedAt: scanning.CreatedAt, + UpdatedAt: scanning.UpdatedAt, + }) + } + return resp, total, nil +} + +func GetScanningModuleByID(id string, log *zap.SugaredLogger) (*Scanning, error) { + scanning, err := commonrepo.NewScanningColl().GetByID(id) + if err != nil { + log.Errorf("failed to get scanning from mongodb, the error is: %s", err) + return nil, err + } + return ConvertDBScanningModule(scanning), nil +} + +func DeleteScanningModuleByID(id string, log *zap.SugaredLogger) error { + scanning, err := commonrepo.NewScanningColl().GetByID(id) + if err != nil { + log.Errorf("failed to get scanning from mongodb, the error is: %s", err) + return err + } + + err = commonservice.ProcessWebhook(nil, scanning.AdvancedSetting.HookCtl.Items, webhook.ScannerPrefix+scanning.Name, log) + if err != nil { + log.Errorf("failed to process webhook for scanning module: %s, the error is: %s", id, err) + return err + } + + err = commonrepo.NewScanningColl().DeleteByID(id) + if err != nil { + log.Errorf("failed to delete scanning from mongodb, the error is: %s", err) + } + return err +} + +// CreateScanningTask uses notificationID if the task is triggered by webhook, otherwise it should be empty +func CreateScanningTask(id string, req []*ScanningRepoInfo, notificationID, username string, log *zap.SugaredLogger) (int64, error) { + scanningInfo, err := commonrepo.NewScanningColl().GetByID(id) + if err != nil { + log.Errorf("failed to get scanning from mongodb, the error is: %s", err) + return 0, err + } + + scanningName := fmt.Sprintf("%s-%s-%s", scanningInfo.Name, id, "scanning-job") + + nextTaskID, err := commonrepo.NewCounterColl().GetNextSeq(fmt.Sprintf(setting.ScanningTaskFmt, scanningName)) + if err != nil { + log.Errorf("failed to generated task id for scanning task, error: %s", err) + return 0, e.ErrGetCounter.AddDesc(err.Error()) + } + + imageInfo, err := commonrepo.NewBasicImageColl().Find(scanningInfo.ImageID) + if err != nil { + log.Errorf("failed to get image information to create scanning task, error: %s", err) + return 0, err + } + + scanningImage := imageInfo.Value + + if imageInfo.ImageFrom == commonmodels.ImageFromKoderover { + scanningImage = strings.ReplaceAll(config.ReaperImage(), "${BuildOS}", imageInfo.Value) + } + + registries, err := commonservice.ListRegistryNamespaces("", true, log) + if err != nil { + log.Errorf("ListRegistryNamespaces err:%v", err) + return 0, err + } + + repos := make([]*types.Repository, 0) + for _, arg := range req { + rep, err := systemconfig.New().GetCodeHost(arg.CodehostID) + if err != nil { + log.Errorf("failed to get codehost info from mongodb, the error is: %s", err) + return 0, err + } + + repoInfo := &types.Repository{ + Source: rep.Type, + RepoOwner: arg.RepoOwner, + RepoName: arg.RepoName, + Branch: arg.Branch, + PR: arg.PR, + CodehostID: arg.CodehostID, + OauthToken: rep.AccessToken, + Address: rep.Address, + Username: rep.Username, + Password: rep.Password, + EnableProxy: rep.EnableProxy, + RepoNamespace: arg.RepoNamespace, + Tag: arg.Tag, + } + + for _, repo := range scanningInfo.Repos { + // make sure we are using the same repo's configuration + if repo.CodehostID == arg.CodehostID && repo.RepoName == arg.RepoName { + repoInfo.SubModules = repo.SubModules + repoInfo.RemoteName = repo.RemoteName + repoInfo.CheckoutPath = repo.CheckoutPath + break + } + } + + repos = append(repos, repoInfo) + } + + scanningTask := &task.Scanning{ + TaskType: config.TaskScanning, + Status: config.StatusCreated, + ScanningID: scanningInfo.ID.Hex(), + Name: scanningInfo.Name, + ImageInfo: scanningImage, + ResReq: scanningInfo.AdvancedSetting.ResReq, + ResReqSpec: scanningInfo.AdvancedSetting.ResReqSpec, + Registries: registries, + Parameter: scanningInfo.Parameter, + Script: scanningInfo.Script, + Timeout: DefaultScanningTimeout, + ClusterID: scanningInfo.AdvancedSetting.ClusterID, + Repos: repos, + } + + if scanningInfo.ScannerType == "sonarQube" { + sonarInfo, err := commonrepo.NewSonarIntegrationColl().GetByID(context.TODO(), scanningInfo.SonarID) + if err != nil { + log.Errorf("failed to get sonar integration information to create scanning task, error: %s", err) + return 0, err + } + + scanningTask.SonarInfo = &types.SonarInfo{ + Token: sonarInfo.Token, + ServerAddress: sonarInfo.ServerAddress, + } + } + + proxies, err := commonrepo.NewProxyColl().List(&commonrepo.ProxyArgs{}) + if err != nil { + log.Errorf("failed to get proxy info to create scanning task, error: %s", err) + return 0, err + } + if len(proxies) != 0 { + scanningTask.Proxy = proxies[0] + } + + scanningSubtask, err := scanningTask.ToSubTask() + if err != nil { + log.Errorf("failed to convert scanning subtask, error: %s", err) + return 0, e.ErrCreateTask.AddDesc(err.Error()) + } + + stages := make([]*commonmodels.Stage, 0) + workflowservice.AddSubtaskToStage(&stages, scanningSubtask, scanningInfo.Name) + sort.Sort(workflowservice.ByStageKind(stages)) + + configPayload := commonservice.GetConfigPayload(0) + + defaultS3, err := s3.FindDefaultS3() + if err != nil { + log.Errorf("cannot find the default s3 to store the logs, error: %s", err) + return 0, e.ErrFindDefaultS3Storage.AddDesc("default storage is required by distribute task") + } + + defaultURL, err := defaultS3.GetEncryptedURL() + if err != nil { + log.Errorf("cannot convert the s3 config to an encrypted URI, error: %s", err) + return 0, e.ErrS3Storage.AddErr(err) + } + + finalTask := &task.Task{ + TaskID: nextTaskID, + ProductName: scanningInfo.ProjectName, + PipelineName: scanningName, + Type: config.ScanningType, + Status: config.StatusCreated, + TaskCreator: username, + CreateTime: time.Now().Unix(), + Stages: stages, + ConfigPayload: configPayload, + StorageURI: defaultURL, + ScanningArgs: &commonmodels.ScanningArgs{ + ScanningName: scanningInfo.Name, + ScanningID: scanningInfo.ID.Hex(), + NotificationID: notificationID, + }, + } + + if len(finalTask.Stages) <= 0 { + return 0, e.ErrCreateTask.AddDesc(e.PipelineSubTaskNotFoundErrMsg) + } + + if err := workflowservice.CreateTask(finalTask); err != nil { + log.Error(err) + return 0, e.ErrCreateTask + } + + // Updating the comment in the git repository, this will not cause the function to return error if this function call fails + err = scmnotify.NewService().UpdateWebhookCommentForScanning(finalTask, log) + if err != nil { + log.Warnf("Failed to update comment for scanning: %s, the error is: %s", scanningInfo.ID.Hex(), err) + } + + return nextTaskID, nil +} + +func ListScanningTask(id string, pageNum, pageSize int64, log *zap.SugaredLogger) (*ListScanningTaskResp, error) { + scanningInfo, err := commonrepo.NewScanningColl().GetByID(id) + if err != nil { + log.Errorf("failed to get scanning from mongodb, the error is: %s", err) + return nil, err + } + + scanningName := fmt.Sprintf("%s-%s-%s", scanningInfo.Name, id, "scanning-job") + listTaskOpt := &commonrepo.ListTaskOption{ + PipelineName: scanningName, + Limit: int(pageSize), + Skip: int((pageNum - 1) * pageSize), + Detail: true, + Type: config.ScanningType, + } + countTaskOpt := &commonrepo.CountTaskOption{ + PipelineNames: []string{scanningName}, + Type: config.ScanningType, + } + resp, err := commonrepo.NewTaskColl().List(listTaskOpt) + if err != nil { + log.Errorf("failed to list scanning task for scanning: %s, the error is: %s", id, err) + return nil, err + } + cnt, err := commonrepo.NewTaskColl().Count(countTaskOpt) + if err != nil { + log.Errorf("failed to count scanning task for scanning: %s, the error is: %s", id, err) + return nil, err + } + + scanTasks := make([]*ScanningTaskResp, 0) + + for _, scanningTask := range resp { + taskInfo := &ScanningTaskResp{ + ScanID: scanningTask.TaskID, + Status: string(scanningTask.Status), + Creator: scanningTask.TaskCreator, + CreatedAt: scanningTask.CreateTime, + } + if scanningTask.Status == config.StatusPassed || scanningTask.Status == config.StatusCancelled || scanningTask.Status == config.StatusFailed { + taskInfo.RunTime = scanningTask.EndTime - scanningTask.StartTime + } + scanTasks = append(scanTasks, taskInfo) + } + + return &ListScanningTaskResp{ + ScanInfo: &ScanningInfo{ + Editor: scanningInfo.UpdatedBy, + UpdatedAt: scanningInfo.UpdatedAt, + }, + ScanTasks: scanTasks, + TotalTasks: cnt, + }, nil +} + +func GetScanningTaskInfo(scanningID string, taskID int64, log *zap.SugaredLogger) (*ScanningTaskDetail, error) { + scanningInfo, err := commonrepo.NewScanningColl().GetByID(scanningID) + if err != nil { + log.Errorf("failed to get scanning from mongodb, the error is: %s", err) + return nil, err + } + + scanningName := fmt.Sprintf("%s-%s-%s", scanningInfo.Name, scanningID, "scanning-job") + resp, err := commonrepo.NewTaskColl().Find(taskID, scanningName, config.ScanningType) + if err != nil { + log.Errorf("failed to get task information, error: %s", err) + return nil, err + } + + resultAddr := "" + + if scanningInfo.ScannerType == "sonarQube" { + sonarInfo, err := commonrepo.NewSonarIntegrationColl().GetByID(context.TODO(), scanningInfo.SonarID) + if err != nil { + log.Errorf("failed to get sonar integration info, error: %s", err) + return nil, err + } + resultAddr = sonarInfo.ServerAddress + } + + repoInfo := resp.Stages[0].SubTasks[scanningInfo.Name] + scanningTaskInfo, err := base.ToScanning(repoInfo) + if err != nil { + log.Errorf("failed to convert the content into scanning subtask, the error is: %s", err) + return nil, fmt.Errorf("failed to convert the content into scanning subtask, the error is: %s", err) + } + + // for security reasons, we set all sensitive information to empty + for _, repo := range scanningTaskInfo.Repos { + repo.OauthToken = "" + repo.Password = "" + repo.Username = "" + } + + return &ScanningTaskDetail{ + Creator: resp.TaskCreator, + Status: string(resp.Status), + CreateTime: resp.CreateTime, + EndTime: resp.EndTime, + RepoInfo: scanningTaskInfo.Repos, + ResultLink: resultAddr, + }, nil +} + +func CancelScanningTask(userName, scanningID string, taskID int64, typeString config.PipelineType, requestID string, log *zap.SugaredLogger) error { + scanningInfo, err := commonrepo.NewScanningColl().GetByID(scanningID) + if err != nil { + log.Errorf("failed to get scanning from mongodb, the error is: %s", err) + return err + } + + scanningTaskName := fmt.Sprintf("%s-%s-%s", scanningInfo.Name, scanningID, "scanning-job") + + return commonservice.CancelTaskV2(userName, scanningTaskName, taskID, typeString, requestID, log) +} diff --git a/pkg/microservice/aslan/core/workflow/testing/service/test_task.go b/pkg/microservice/aslan/core/workflow/testing/service/test_task.go index 30cbb7636eaaebaf399fd1087bf54ddfd39a6fb3..af206085576674a01e5e24186b535c9e27815719 100644 --- a/pkg/microservice/aslan/core/workflow/testing/service/test_task.go +++ b/pkg/microservice/aslan/core/workflow/testing/service/test_task.go @@ -57,7 +57,6 @@ func CreateTestTask(args *commonmodels.TestTaskArgs, log *zap.SugaredLogger) (*C return nil, e.ErrGetCounter.AddDesc(err.Error()) } - // 获取全局configpayload configPayload := commonservice.GetConfigPayload(0) defaultS3Store, err := s3.FindDefaultS3() diff --git a/pkg/microservice/aslan/core/workflow/testing/service/testing.go b/pkg/microservice/aslan/core/workflow/testing/service/testing.go index 0ea62ff57848a648f84e91b767f2fa3524335fc4..2c8dae306f633dc1cfb938dbb0ae835f9ea70e3b 100644 --- a/pkg/microservice/aslan/core/workflow/testing/service/testing.go +++ b/pkg/microservice/aslan/core/workflow/testing/service/testing.go @@ -32,6 +32,8 @@ import ( commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/nsq" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/webhook" + commonutil "github.com/koderover/zadig/pkg/microservice/aslan/core/common/util" workflowservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" "github.com/koderover/zadig/pkg/setting" e "github.com/koderover/zadig/pkg/tool/errors" @@ -43,12 +45,19 @@ func CreateTesting(username string, testing *commonmodels.Testing, log *zap.Suga if len(testing.Name) == 0 { return e.ErrCreateTestModule.AddDesc("empty Name") } - + if err := commonutil.CheckDefineResourceParam(testing.PreTest.ResReq, testing.PreTest.ResReqSpec); err != nil { + return e.ErrCreateTestModule.AddDesc(err.Error()) + } err := HandleCronjob(testing, log) if err != nil { return e.ErrCreateTestModule.AddErr(err) } + err = commonservice.ProcessWebhook(testing.HookCtl.Items, nil, webhook.TestingPrefix+testing.Name, log) + if err != nil { + return e.ErrUpdateTestModule.AddErr(err) + } + testing.UpdateBy = username err = commonrepo.NewTestingColl().Create(testing) if err != nil { @@ -96,7 +105,9 @@ func UpdateTesting(username string, testing *commonmodels.Testing, log *zap.Suga if len(testing.Name) == 0 { return e.ErrUpdateTestModule.AddDesc("empty Name") } - + if err := commonutil.CheckDefineResourceParam(testing.PreTest.ResReq, testing.PreTest.ResReqSpec); err != nil { + return e.ErrUpdateTestModule.AddDesc(err.Error()) + } err := HandleCronjob(testing, log) if err != nil { return e.ErrUpdateTestModule.AddErr(err) @@ -107,6 +118,11 @@ func UpdateTesting(username string, testing *commonmodels.Testing, log *zap.Suga commonservice.EnsureSecretEnvs(existed.PreTest.Envs, testing.PreTest.Envs) } + err = commonservice.ProcessWebhook(testing.HookCtl.Items, existed.HookCtl.Items, webhook.TestingPrefix+testing.Name, log) + if err != nil { + return e.ErrUpdateTestModule.AddErr(err) + } + testing.UpdateBy = username testing.UpdateTime = time.Now().Unix() @@ -271,54 +287,6 @@ func ListCronjob(name, jobType string) ([]*commonmodels.Cronjob, error) { }) } -func DeleteTestModule(name, productName, requestID string, log *zap.SugaredLogger) error { - opt := new(commonrepo.ListQueueOption) - taskQueue, err := commonrepo.NewQueueColl().List(opt) - if err != nil { - log.Errorf("List queued task error: %v", err) - return fmt.Errorf("list queued task error: %v", err) - } - pipelineName := fmt.Sprintf("%s-%s", name, "job") - // 当task还在运行时，先取消任务 - for _, task := range taskQueue { - if task.PipelineName == pipelineName && task.Type == config.TestType { - if err = commonservice.CancelTaskV2("system", task.PipelineName, task.TaskID, config.TestType, requestID, log); err != nil { - log.Errorf("test task still running,cancel pipeline %s task %d", task.PipelineName, task.TaskID) - } - } - } - - return DeleteTestingModule(name, productName, log) -} - -func DeleteTestingModule(name, productName string, log *zap.SugaredLogger) error { - if len(name) == 0 { - return e.ErrDeleteTestModule.AddDesc("empty Name") - } - - err := commonrepo.NewTestingColl().Delete(name, productName) - if err != nil { - log.Errorf("[Testing.Delete] %s error: %v", name, err) - return e.ErrDeleteTestModule.AddErr(err) - } - - if err := commonrepo.NewTaskColl().DeleteByPipelineNameAndType(fmt.Sprintf("%s-%s", name, "job"), config.TestType); err != nil { - log.Errorf("[Testing.Delete] PipelineTaskV2.DeleteByPipelineNameAndType test %s error: %v", name, err) - } - - if err := commonrepo.NewTestTaskStatColl().Delete(name); err != nil { - log.Errorf("[TestTaskStat.Delete] %s error: %v", name, err) - } - - pipelineName := fmt.Sprintf("%s-%s", name, "job") - counterName := fmt.Sprintf(setting.TestTaskFmt, pipelineName) - if err := commonrepo.NewCounterColl().Delete(counterName); err != nil { - log.Errorf("[Counter.Delete] counterName:%s, error: %v", counterName, err) - } - - return nil -} - func GetHTMLTestReport(pipelineName, pipelineType, taskIDStr, testName string, log *zap.SugaredLogger) (string, error) { if err := validateTestReportParam(pipelineName, pipelineType, taskIDStr, testName, log); err != nil { return "", e.ErrGetTestReport.AddErr(err) @@ -330,9 +298,9 @@ func GetHTMLTestReport(pipelineName, pipelineType, taskIDStr, testName string, l return "", e.ErrGetTestReport.AddDesc("invalid taskID") } - task, err := commonrepo.NewTaskColl().Find(taskID, pipelineName, config.WorkflowType) + task, err := commonrepo.NewTaskColl().Find(taskID, pipelineName, config.PipelineType(pipelineType)) if err != nil { - log.Errorf("find task failed, pipelineName: %s, type: %s, taskID: %s, err: %s", pipelineName, config.WorkflowType, taskIDStr, err) + log.Errorf("find task failed, pipelineName: %s, type: %s, taskID: %s, err: %s", pipelineName, config.PipelineType(pipelineType), taskIDStr, err) return "", e.ErrGetTestReport.AddErr(err) } @@ -405,7 +373,7 @@ func validateTestReportParam(pipelineName, pipelineType, taskIDStr, testName str return fmt.Errorf("testName cannot be empty") } - if pipelineType != string(config.WorkflowType) { + if pipelineType != string(config.WorkflowType) && pipelineType != string(config.TestType) { log.Warn("pipelineType is invalid") return fmt.Errorf("pipelineType is invalid") } diff --git a/pkg/microservice/aslan/core/workflow/testing/service/types.go b/pkg/microservice/aslan/core/workflow/testing/service/types.go new file mode 100644 index 0000000000000000000000000000000000000000..1cdb731237d565ed8d20fe11b74b201c6a54bfc0 --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/testing/service/types.go @@ -0,0 +1,127 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/types" +) + +const DefaultScanningTimeout = 60 * 60 + +type Scanning struct { + ID string `json:"id"` + Name string `json:"name"` + ProjectName string `json:"project_name"` + Description string `json:"description"` + ScannerType string `json:"scanner_type"` + ImageID string `json:"image_id"` + SonarID string `json:"sonar_id"` + Repos []*types.Repository `json:"repos"` + // Parameter is for sonarQube type only + Parameter string `json:"parameter"` + // Script is for other type only + Script string `json:"script"` + AdvancedSetting *types.ScanningAdvancedSetting `json:"advanced_settings"` +} + +type ListScanningRespItem struct { + ID string `json:"id"` + Name string `json:"name"` + Statistics *ScanningStatistic `json:"statistics"` + CreatedAt int64 `json:"created_at"` + UpdatedAt int64 `json:"updated_at"` +} + +type ScanningRepoInfo struct { + CodehostID int `json:"codehost_id"` + Source string `json:"source"` + RepoOwner string `json:"repo_owner"` + RepoNamespace string `json:"repo_namespace"` + RepoName string `json:"repo_name"` + PR int `json:"pr"` + Branch string `json:"branch"` + Tag string `json:"tag"` +} + +type ScanningStatistic struct { + TimesRun int64 `json:"times_run"` + AverageRuntime int64 `json:"run_time_average"` +} + +type ListScanningTaskResp struct { + ScanInfo *ScanningInfo `json:"scan_info"` + ScanTasks []*ScanningTaskResp `json:"scan_tasks"` + TotalTasks int `json:"total_tasks"` +} + +type ScanningInfo struct { + Editor string `json:"editor"` + UpdatedAt int64 `json:"updated_at"` +} + +type ScanningTaskResp struct { + ScanID int64 `json:"scan_id"` + Status string `json:"status"` + RunTime int64 `json:"run_time"` + Creator string `json:"creator"` + CreatedAt int64 `json:"created_at"` +} + +type ScanningTaskDetail struct { + Creator string `json:"creator"` + Status string `json:"status"` + CreateTime int64 `json:"create_time"` + EndTime int64 `json:"end_time"` + RepoInfo []*types.Repository `json:"repo_info"` + ResultLink string `json:"result_link,omitempty"` +} + +func ConvertToDBScanningModule(args *Scanning) *commonmodels.Scanning { + // ID is omitted since they are of different type and there will be no use of it + return &commonmodels.Scanning{ + Name: args.Name, + ProjectName: args.ProjectName, + Description: args.Description, + ScannerType: args.ScannerType, + ImageID: args.ImageID, + SonarID: args.SonarID, + Repos: args.Repos, + Parameter: args.Parameter, + Script: args.Script, + AdvancedSetting: args.AdvancedSetting, + } +} + +func ConvertDBScanningModule(scanning *commonmodels.Scanning) *Scanning { + for _, repo := range scanning.Repos { + repo.RepoNamespace = repo.GetRepoNamespace() + } + return &Scanning{ + ID: scanning.ID.Hex(), + Name: scanning.Name, + ProjectName: scanning.ProjectName, + Description: scanning.Description, + ScannerType: scanning.ScannerType, + ImageID: scanning.ImageID, + SonarID: scanning.SonarID, + Repos: scanning.Repos, + Parameter: scanning.Parameter, + Script: scanning.Script, + AdvancedSetting: scanning.AdvancedSetting, + } +} diff --git a/pkg/microservice/aslan/core/workflow/testing/service/workspace.go b/pkg/microservice/aslan/core/workflow/testing/service/workspace.go index 7f7916129f7182a33b50a4230d1b2782f8c96083..7e78d86c098d834b8b882369c0367e1aa72aedf7 100644 --- a/pkg/microservice/aslan/core/workflow/testing/service/workspace.go +++ b/pkg/microservice/aslan/core/workflow/testing/service/workspace.go @@ -17,23 +17,35 @@ limitations under the License. package service import ( + "bytes" "fmt" + "io/ioutil" + "os" + "os/exec" + "path" + "strings" "go.uber.org/zap" - commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3" "github.com/koderover/zadig/pkg/setting" s3tool "github.com/koderover/zadig/pkg/tool/s3" + "github.com/koderover/zadig/pkg/util/fs" ) -func GetTestArtifactInfo(pipelineName, dir string, taskID int64, log *zap.SugaredLogger) (*commonmodels.S3Storage, []string, error) { +type GetTestArtifactInfoResp struct { + FileNames []string `json:"file_names"` + NotHistoryFileFlag bool `json:"not_history_file_flag"` +} + +func GetTestArtifactInfo(pipelineName, dir string, taskID int64, log *zap.SugaredLogger) (*GetTestArtifactInfoResp, error) { + resp := new(GetTestArtifactInfoResp) fis := make([]string, 0) storage, err := s3.FindDefaultS3() if err != nil { log.Errorf("GetTestArtifactInfo FindDefaultS3 err:%v", err) - return nil, fis, nil + return resp, err } if storage.Subfolder != "" { storage.Subfolder = fmt.Sprintf("%s/%s/%d/%s", storage.Subfolder, pipelineName, taskID, "artifact") @@ -47,13 +59,70 @@ func GetTestArtifactInfo(pipelineName, dir string, taskID int64, log *zap.Sugare client, err := s3tool.NewClient(storage.Endpoint, storage.Ak, storage.Sk, storage.Insecure, forcedPathStyle) if err != nil { log.Errorf("GetTestArtifactInfo create s3 client err:%v", err) - return nil, fis, nil + return resp, err + } + + objectKey := storage.GetObjectPath(fmt.Sprintf("%s/%s/%s", dir, "workspace", setting.ArtifactResultOut)) + object, err := client.GetFile(storage.Bucket, objectKey, &s3tool.DownloadOption{IgnoreNotExistError: true, RetryNum: 2}) + if err != nil { + log.Errorf("GetTestArtifactInfo GetFile err:%s", err) + return resp, err + } + if object != nil && *object.ContentLength > 0 { + tempDir, _ := ioutil.TempDir("", "") + sourcePath := path.Join(tempDir, "artifact") + if _, err := os.Stat(sourcePath); os.IsNotExist(err) { + _ = os.MkdirAll(sourcePath, 0777) + } + + file, err := os.Create(path.Join(sourcePath, setting.ArtifactResultOut)) + if err != nil { + return nil, fmt.Errorf("failed to create file %s %s", setting.ArtifactResultOut, err) + } + defer func() { + _ = file.Close() + _ = os.Remove(tempDir) + }() + + err = fs.SaveFile(object.Body, file.Name()) + if err != nil { + log.Errorf("Failed to save file to %s, err: %s", file.Name(), err) + return resp, err + } + + cmdtf := exec.Command("tar", "-tf", file.Name()) + var stdoutBuf bytes.Buffer + cmdtf.Stdout = &stdoutBuf + cmdtf.Stderr = os.Stderr + if err = cmdtf.Run(); err != nil { + fmt.Printf("failed to tar -tf err:%s", err) + return resp, err + } + for _, tarFile := range strings.Split(string(stdoutBuf.Bytes()), "\n") { + if strings.HasSuffix(tarFile, "/") { + continue + } + if strings.TrimSpace(tarFile) == "" { + continue + } + fis = append(fis, tarFile) + } + resp.FileNames = fis + resp.NotHistoryFileFlag = true + return resp, nil } + prefix := storage.GetObjectPath(dir) files, err := client.ListFiles(storage.Bucket, prefix, true) if err != nil || len(files) <= 0 { - log.Errorf("GetTestArtifactInfo ListFiles err:%v", err) - return nil, fis, nil + log.Errorf("GetTestArtifactInfo ListFiles err:%s", err) + return resp, err + } + + for _, file := range files { + _, fileName := path.Split(file) + fis = append(fis, fileName) } - return storage.S3Storage, files, nil + resp.FileNames = fis + return resp, nil } diff --git a/pkg/microservice/aslan/server/rest/router.go b/pkg/microservice/aslan/server/rest/router.go index 1339a6f96a8e3f375bfebe57569a2d04fab7e49f..8b1d62c3ebea5ac62ff211a81040a33dacf0ce0e 100644 --- a/pkg/microservice/aslan/server/rest/router.go +++ b/pkg/microservice/aslan/server/rest/router.go @@ -17,28 +17,28 @@ limitations under the License. package rest import ( - "github.com/gin-contrib/sessions" - "github.com/gin-contrib/sessions/cookie" "github.com/gin-gonic/gin" swaggerfiles "github.com/swaggo/files" ginswagger "github.com/swaggo/gin-swagger" + cachehandler "github.com/koderover/zadig/pkg/handler/cache" buildhandler "github.com/koderover/zadig/pkg/microservice/aslan/core/build/handler" codehosthandler "github.com/koderover/zadig/pkg/microservice/aslan/core/code/handler" + collaborationhandler "github.com/koderover/zadig/pkg/microservice/aslan/core/collaboration/handler" commonhandler "github.com/koderover/zadig/pkg/microservice/aslan/core/common/handler" cronhandler "github.com/koderover/zadig/pkg/microservice/aslan/core/cron/handler" deliveryhandler "github.com/koderover/zadig/pkg/microservice/aslan/core/delivery/handler" environmenthandler "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/handler" + labelhandler "github.com/koderover/zadig/pkg/microservice/aslan/core/label/handler" loghandler "github.com/koderover/zadig/pkg/microservice/aslan/core/log/handler" multiclusterhandler "github.com/koderover/zadig/pkg/microservice/aslan/core/multicluster/handler" projecthandler "github.com/koderover/zadig/pkg/microservice/aslan/core/project/handler" servicehandler "github.com/koderover/zadig/pkg/microservice/aslan/core/service/handler" - settinghandler "github.com/koderover/zadig/pkg/microservice/aslan/core/setting/handler" + stathandler "github.com/koderover/zadig/pkg/microservice/aslan/core/stat/handler" systemhandler "github.com/koderover/zadig/pkg/microservice/aslan/core/system/handler" templatehandler "github.com/koderover/zadig/pkg/microservice/aslan/core/templatestore/handler" workflowhandler "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/handler" testinghandler "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/testing/handler" - gin2 "github.com/koderover/zadig/pkg/middleware/gin" // Note: have to load docs for swagger to work. See https://blog.csdn.net/weixin_43249914/article/details/103035711 _ "github.com/koderover/zadig/pkg/microservice/aslan/server/rest/doc" @@ -54,14 +54,6 @@ import ( // @license.url http://www.apache.org/licenses/LICENSE-2.0.html // @BasePath /api/aslan func (s *engine) injectRouterGroup(router *gin.RouterGroup) { - store := cookie.NewStore([]byte("C12f4d5957k345e9g6e86c117e1bfu5kndjevf8u")) - cookieOption := sessions.Options{ - Path: "/", - HttpOnly: true, - } - store.Options(cookieOption) - router.Use(sessions.Sessions("ASLAN", store)) - Auth := gin2.Auth() // --------------------------------------------------------------------------------------- // 对外公共接口 // --------------------------------------------------------------------------------------- @@ -72,32 +64,32 @@ func (s *engine) injectRouterGroup(router *gin.RouterGroup) { s.HandleContext(c) }) public.GET("/health", commonhandler.Health) + public.POST("/callback", commonhandler.HandleCallback) } - // no auth required, should not be exposed via poetry-api-proxy or will fail + // no auth required router.GET("/api/hub/connect", multiclusterhandler.ClusterConnectFromAgent) - router.GET("/api/kodespace/downloadUrl", Auth, commonhandler.GetToolDownloadURL) + router.GET("/api/kodespace/downloadUrl", commonhandler.GetToolDownloadURL) - jwt := router.Group("/api/token", Auth) - { - jwt.GET("", commonhandler.GetToken) - } for name, r := range map[string]injector{ - "/api/project": new(projecthandler.Router), - "/api/code": new(codehosthandler.Router), - "/api/system": new(systemhandler.Router), - "/api/service": new(servicehandler.Router), - "/api/setting": new(settinghandler.Router), - "/api/environment": new(environmenthandler.Router), - "/api/cron": new(cronhandler.Router), - "/api/workflow": new(workflowhandler.Router), - "/api/build": new(buildhandler.Router), - "/api/delivery": new(deliveryhandler.Router), - "/api/logs": new(loghandler.Router), - "/api/testing": new(testinghandler.Router), - "/api/cluster": new(multiclusterhandler.Router), - "/api/template": new(templatehandler.Router), + "/api/project": new(projecthandler.Router), + "/api/code": new(codehosthandler.Router), + "/api/system": new(systemhandler.Router), + "/api/service": new(servicehandler.Router), + "/api/environment": new(environmenthandler.Router), + "/api/cron": new(cronhandler.Router), + "/api/workflow": new(workflowhandler.Router), + "/api/build": new(buildhandler.Router), + "/api/delivery": new(deliveryhandler.Router), + "/api/logs": new(loghandler.Router), + "/api/testing": new(testinghandler.Router), + "/api/cluster": new(multiclusterhandler.Router), + "/api/template": new(templatehandler.Router), + "/api/collaboration": new(collaborationhandler.Router), + "/api/label": new(labelhandler.Router), + "/api/stat": new(stathandler.Router), + "/api/cache": cachehandler.NewRouter(), } { r.Inject(router.Group(name)) } diff --git a/pkg/microservice/aslan/server/rest/server.go b/pkg/microservice/aslan/server/rest/server.go index 18843b0a5ead8f194d217a281527d5152e939795..04fdce0bfd669f5721d41ef3276ab0fbc489b616 100644 --- a/pkg/microservice/aslan/server/rest/server.go +++ b/pkg/microservice/aslan/server/rest/server.go @@ -52,9 +52,11 @@ func (s *engine) injectMiddlewares() { if s.mode == gin.TestMode { return } + g.Use(ginmiddleware.OperationLogStatus()) g.Use(ginmiddleware.Response()) g.Use(ginmiddleware.RequestID()) g.Use(ginmiddleware.RequestLog(log.NewFileLogger(config.RequestLogFile()))) + g.Use(ginmiddleware.GetCollaborationNew()) g.Use(gin.Recovery()) } diff --git a/pkg/microservice/cron/config/config.go b/pkg/microservice/cron/config/config.go index b044a63338fab341e9fe8ef80cb4c86576e80189..95f8c0805a070298ba0a14312a5c440aa0e5118d 100644 --- a/pkg/microservice/cron/config/config.go +++ b/pkg/microservice/cron/config/config.go @@ -25,18 +25,10 @@ import ( "github.com/koderover/zadig/pkg/setting" ) -func RootToken() string { - return viper.GetString(setting.ENVRootToken) -} - func CollieAPI() string { return configbase.CollieServiceAddress() } -func CollieToken() string { - return RootToken() -} - func NsqLookupAddrs() []string { return strings.Split(viper.GetString(setting.ENVNsqLookupAddrs), ",") } diff --git a/pkg/microservice/cron/core/service/client/caller.go b/pkg/microservice/cron/core/service/client/caller.go index dca82f3794b8e40a66cf76e7974ea1371164c161..2f144cd1418faa61f9beef355aeeb3b128a75f26 100644 --- a/pkg/microservice/cron/core/service/client/caller.go +++ b/pkg/microservice/cron/core/service/client/caller.go @@ -24,8 +24,6 @@ import ( "net/http" "go.uber.org/zap" - - "github.com/koderover/zadig/pkg/setting" ) func (c *Client) ScheduleCall(api string, args interface{}, log *zap.SugaredLogger) error { @@ -41,7 +39,7 @@ func (c *Client) ScheduleCall(api string, args interface{}, log *zap.SugaredLogg log.Errorf("create post request error : %v", err) return err } - request.Header.Set("Authorization", fmt.Sprintf("%s %s", setting.TIMERAPIKEY, c.Token)) + var resp *http.Response resp, err = c.Conn.Do(request) if err == nil { diff --git a/pkg/microservice/cron/core/service/client/client.go b/pkg/microservice/cron/core/service/client/client.go index 7d41290cf7eadc311df6233a4caaa42027bfcad5..ab416baaffa0233dc1b803c6dcbdc1e92f0d0abf 100644 --- a/pkg/microservice/cron/core/service/client/client.go +++ b/pkg/microservice/cron/core/service/client/client.go @@ -23,17 +23,15 @@ import ( type Client struct { APIBase string - Token string Conn *http.Client } // NewAslanClient is to get aslan client func -func NewAslanClient(host, token string) *Client { +func NewAslanClient(host string) *Client { jar, _ := cookiejar.New(nil) c := &Client{ APIBase: host, - Token: token, Conn: &http.Client{Transport: http.DefaultTransport, Jar: jar}, } diff --git a/pkg/microservice/cron/core/service/client/collie_client.go b/pkg/microservice/cron/core/service/client/collie_client.go index 413a8151dbb083c59ae8997e6d8feeef00ae4950..dbf6a1182898e735e7c6f2a13a58573d827b4732 100644 --- a/pkg/microservice/cron/core/service/client/collie_client.go +++ b/pkg/microservice/cron/core/service/client/collie_client.go @@ -17,27 +17,22 @@ limitations under the License. package client import ( - "github.com/koderover/zadig/pkg/setting" "github.com/koderover/zadig/pkg/tool/httpclient" ) type CollieClient struct { *httpclient.Client - host string - token string + host string } -func NewCollieClient(host, token string) *CollieClient { +func NewCollieClient(host string) *CollieClient { c := httpclient.New( - httpclient.SetAuthScheme(setting.RootAPIKey), - httpclient.SetAuthToken(token), httpclient.SetHostURL(host), ) return &CollieClient{ Client: c, host: host, - token: token, } } diff --git a/pkg/microservice/cron/core/service/client/cronjob.go b/pkg/microservice/cron/core/service/client/cronjob.go index 2509b2f500708fb122d7fbfed84e051a3c9fe08b..42bb62a4048daa7bc92253c20a3ac4cd81a366f5 100644 --- a/pkg/microservice/cron/core/service/client/cronjob.go +++ b/pkg/microservice/cron/core/service/client/cronjob.go @@ -24,12 +24,12 @@ import ( "io" "io/ioutil" "net/http" + "path" "time" "go.uber.org/zap" "github.com/koderover/zadig/pkg/microservice/cron/core/service" - "github.com/koderover/zadig/pkg/setting" "github.com/koderover/zadig/pkg/tool/rsa" ) @@ -67,6 +67,17 @@ func (c *Client) TriggerCleanProducts(log *zap.SugaredLogger) error { return err } +// TriggerCleanCIResources trigger clean CollaborationInstance Resources +func (c *Client) TriggerCleanCIResources(log *zap.SugaredLogger) error { + url := fmt.Sprintf("%s/collaboration/collaborations/cron/clean", c.APIBase) + log.Info("start clean CollaborationInstance Resources..") + err := c.sendRequest(url) + if err != nil { + log.Errorf("trigger clean CollaborationInstance Resources error :%s", err) + } + return err +} + // RunPipelineTask ... func (c *Client) RunPipelineTask(args *service.TaskArgs, log *zap.SugaredLogger) error { url := fmt.Sprintf("%s/workflow/v2/tasks", c.APIBase) @@ -84,7 +95,7 @@ func (c *Client) RunPipelineTask(args *service.TaskArgs, log *zap.SugaredLogger) } func (c *Client) RunWorkflowTask(args *service.WorkflowTaskArgs, log *zap.SugaredLogger) error { - url := fmt.Sprintf("%s/workflow/workflowtask", c.APIBase) + url := path.Join(c.APIBase, "workflow/workflowtask", args.WorkflowName) log.Info("start run scheduled task..") body, err := json.Marshal(args) if err != nil { @@ -169,7 +180,7 @@ func (c *Client) sendRequest(url string) error { if err != nil { return err } - request.Header.Set("Authorization", fmt.Sprintf("%s %s", setting.TIMERAPIKEY, c.Token)) + resp, err := c.Conn.Do(request) if err == nil { defer func() { _ = resp.Body.Close() }() @@ -183,7 +194,6 @@ func (c *Client) sendPostRequest(url string, body io.Reader, log *zap.SugaredLog log.Errorf("create post request error : %v", err) return "", err } - request.Header.Set("Authorization", fmt.Sprintf("%s %s", setting.TIMERAPIKEY, c.Token)) var resp *http.Response resp, err = c.Conn.Do(request) if err != nil { diff --git a/pkg/microservice/cron/core/service/client/env.go b/pkg/microservice/cron/core/service/client/env.go index 468b0190c334d44ea06466891c7ab5c58314bd1a..2bbda456f195f0d5228257627449b97c3e8f64aa 100644 --- a/pkg/microservice/cron/core/service/client/env.go +++ b/pkg/microservice/cron/core/service/client/env.go @@ -22,29 +22,32 @@ import ( "fmt" "io/ioutil" "net/http" + "strings" "github.com/pkg/errors" "go.uber.org/zap" "github.com/koderover/zadig/pkg/microservice/cron/core/service" - "github.com/koderover/zadig/pkg/setting" ) -func (c *Client) ListEnvs(log *zap.SugaredLogger) ([]*service.ProductRevision, error) { +type EvnListOption struct { + BasicFacility string + DeployType []string +} +func (c *Client) ListEnvs(log *zap.SugaredLogger, option *EvnListOption) ([]*service.ProductRevision, error) { var ( err error resp = make([]*service.ProductRevision, 0) ) - url := fmt.Sprintf("%s/environment/revision/products?basicFacility=%s", c.APIBase, "cloud_host") + url := fmt.Sprintf("%s/environment/revision/products?basicFacility=%s&deployType=%s", c.APIBase, option.BasicFacility, strings.Join(option.DeployType, ",")) request, err := http.NewRequest("GET", url, nil) if err != nil { log.Errorf("ListEnvs new http request error: %v", err) return nil, err } - request.Header.Set("Authorization", fmt.Sprintf("%s %s", setting.RootAPIKey, c.Token)) var ret *http.Response if ret, err = c.Conn.Do(request); err == nil { defer func() { _ = ret.Body.Close() }() @@ -59,19 +62,45 @@ func (c *Client) ListEnvs(log *zap.SugaredLogger) ([]*service.ProductRevision, e return resp, errors.WithMessage(err, "failed to list envs") } +func (c *Client) ListEnvResources(productName, envName string, log *zap.SugaredLogger) ([]*service.EnvResource, error) { + var ( + err error + resp = make([]*service.EnvResource, 0) + ) + url := fmt.Sprintf("%s/environment/envcfgs?autoSync=true&projectName=%s&envName=%s", c.APIBase, productName, envName) + request, err := http.NewRequest("GET", url, nil) + if err != nil { + log.Errorf("ListEnvResources new http request error: %s", err) + return nil, err + } + + var ret *http.Response + if ret, err = c.Conn.Do(request); err == nil { + defer func() { _ = ret.Body.Close() }() + var body []byte + body, err = ioutil.ReadAll(ret.Body) + if err == nil { + if err = json.Unmarshal(body, &resp); err == nil { + return resp, nil + } + } + } + + return resp, err +} + func (c *Client) GetEnvService(productName, envName string, log *zap.SugaredLogger) (*service.ProductResp, error) { var ( err error envObj = new(service.ProductResp) ) - url := fmt.Sprintf("%s/environment/environments/%s?envName=%s", c.APIBase, productName, envName) + url := fmt.Sprintf("%s/environment/environments/%s?projectName=%s", c.APIBase, envName, productName) request, err := http.NewRequest("GET", url, nil) if err != nil { log.Errorf("GetService new http request error: %v", err) return nil, err } - request.Header.Set("Authorization", fmt.Sprintf("%s %s", setting.RootAPIKey, c.Token)) var ret *http.Response if ret, err = c.Conn.Do(request); err == nil { defer func() { _ = ret.Body.Close() }() @@ -83,10 +112,36 @@ func (c *Client) GetEnvService(productName, envName string, log *zap.SugaredLogg } } } - return envObj, errors.WithMessage(err, "failed to get service") } +func (c *Client) GetRenderset(name string, revision int64, log *zap.SugaredLogger) (*service.ProductRenderset, error) { + var ( + err error + rendersetObj = new(service.ProductRenderset) + ) + url := fmt.Sprintf("%s/project/renders/render/%s/revision/%d", c.APIBase, name, revision) + request, err := http.NewRequest("GET", url, nil) + if err != nil { + log.Errorf("GetService new http request error: %v", err) + return nil, err + } + + var ret *http.Response + if ret, err = c.Conn.Do(request); err == nil { + defer func() { _ = ret.Body.Close() }() + var body []byte + body, err = ioutil.ReadAll(ret.Body) + if err == nil { + if err = json.Unmarshal(body, &rendersetObj); err == nil { + return rendersetObj, nil + } + } + } + + return rendersetObj, errors.WithMessage(err, "failed to get renderset") +} + func (c *Client) GetService(serviceName, productName, serviceType string, revision int64, log *zap.SugaredLogger) (*service.Service, error) { var ( err error @@ -99,7 +154,6 @@ func (c *Client) GetService(serviceName, productName, serviceType string, revisi return nil, err } - request.Header.Set("Authorization", fmt.Sprintf("%s %s", setting.RootAPIKey, c.Token)) var ret *http.Response if ret, err = c.Conn.Do(request); err == nil { defer func() { _ = ret.Body.Close() }() @@ -129,7 +183,6 @@ func (c *Client) UpdateService(args *service.ServiceTmplObject, log *zap.Sugared return err } - request.Header.Set("Authorization", fmt.Sprintf("%s %s", setting.RootAPIKey, c.Token)) var ret *http.Response if ret, err = c.Conn.Do(request); err == nil { defer func() { _ = ret.Body.Close() }() @@ -142,6 +195,40 @@ func (c *Client) UpdateService(args *service.ServiceTmplObject, log *zap.Sugared return nil } +func (c *Client) SyncEnvVariables(productName, envName string, log *zap.SugaredLogger) error { + url := fmt.Sprintf("%s/environment/environments/%s/syncVariables?projectName=%s", c.APIBase, envName, productName) + request, err := http.NewRequest("PUT", url, nil) + if err != nil { + log.Errorf("SyncEnvVariables new http request error: %v", err) + return err + } + + var ret *http.Response + if ret, err = c.Conn.Do(request); err == nil { + defer func() { _ = ret.Body.Close() }() + _, err := ioutil.ReadAll(ret.Body) + if err != nil { + return errors.WithMessage(err, "failed to sync product variables ") + } + } + return nil +} + +func (c *Client) SyncEnvResource(productName, envName, resType, resName string, log *zap.SugaredLogger) error { + url := fmt.Sprintf("%s/environment/envcfgs/%s/%s/%s/sync?projectName=%s", c.APIBase, envName, resType, resName, productName) + request, err := http.NewRequest("PUT", url, nil) + if err != nil { + log.Errorf("SyncEnvResource new http request error: %v", err) + return err + } + + var ret *http.Response + if ret, err = c.Conn.Do(request); err == nil { + defer func() { _ = ret.Body.Close() }() + } + return nil +} + func (c *Client) GetHostInfo(hostID string, log *zap.SugaredLogger) (*service.PrivateKey, error) { var ( err error @@ -154,7 +241,6 @@ func (c *Client) GetHostInfo(hostID string, log *zap.SugaredLogger) (*service.Pr return nil, err } - request.Header.Set("Authorization", fmt.Sprintf("%s %s", setting.RootAPIKey, c.Token)) var ret *http.Response if ret, err = c.Conn.Do(request); err == nil { defer func() { _ = ret.Body.Close() }() diff --git a/pkg/microservice/cron/core/service/client/operation.go b/pkg/microservice/cron/core/service/client/operation.go index d7f1b4840203c64ff39239c2ad371aed3880d82a..94f2db1bdb7940502bae5f8e66552e7cbc212030 100644 --- a/pkg/microservice/cron/core/service/client/operation.go +++ b/pkg/microservice/cron/core/service/client/operation.go @@ -26,7 +26,6 @@ import ( "go.uber.org/zap" "github.com/koderover/zadig/pkg/microservice/cron/core/service" - "github.com/koderover/zadig/pkg/setting" ) func (c *Client) GetWebHookUser(log *zap.SugaredLogger) (*service.DomainWebHookUser, error) { @@ -42,7 +41,6 @@ func (c *Client) GetWebHookUser(log *zap.SugaredLogger) (*service.DomainWebHookU return domainWebHookUser, err } - request.Header.Set("Authorization", fmt.Sprintf("%s %s", setting.RootAPIKey, c.Token)) var ret *http.Response if ret, err = c.Conn.Do(request); err == nil { defer func() { _ = ret.Body.Close() }() diff --git a/pkg/microservice/cron/core/service/client/pipeline.go b/pkg/microservice/cron/core/service/client/pipeline.go index dacbe0bab9a16e13f567e3d7655a093c9ef5b734..0d1aba96ac95fe7ab39368edeab095fca71ff029 100644 --- a/pkg/microservice/cron/core/service/client/pipeline.go +++ b/pkg/microservice/cron/core/service/client/pipeline.go @@ -26,7 +26,6 @@ import ( "go.uber.org/zap" "github.com/koderover/zadig/pkg/microservice/cron/core/service" - "github.com/koderover/zadig/pkg/setting" ) func (c *Client) ListPipelines(log *zap.SugaredLogger) ([]*service.Pipeline, error) { @@ -40,7 +39,6 @@ func (c *Client) ListPipelines(log *zap.SugaredLogger) ([]*service.Pipeline, err return nil, err } - request.Header.Set("Authorization", fmt.Sprintf("%s %s", setting.RootAPIKey, c.Token)) var ret *http.Response if ret, err = c.Conn.Do(request); err == nil { defer func() { _ = ret.Body.Close() }() diff --git a/pkg/microservice/cron/core/service/client/pm.go b/pkg/microservice/cron/core/service/client/pm.go new file mode 100644 index 0000000000000000000000000000000000000000..7d7dc5b2219fbfb13e0e9f0d0456946340b30288 --- /dev/null +++ b/pkg/microservice/cron/core/service/client/pm.go @@ -0,0 +1,83 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + + "github.com/koderover/zadig/pkg/microservice/cron/core/service" + "github.com/pkg/errors" + "go.uber.org/zap" +) + +func (c *Client) ListPmHosts(log *zap.SugaredLogger) ([]*service.PrivateKeyHosts, error) { + var ( + err error + resp = make([]*service.PrivateKeyHosts, 0) + ) + + url := fmt.Sprintf("%s/system/privateKey/internal", c.APIBase) + request, err := http.NewRequest("GET", url, nil) + if err != nil { + log.Errorf("ListPmHosts new http request error: %v", err) + return nil, err + } + + var ret *http.Response + if ret, err = c.Conn.Do(request); err == nil { + defer func() { _ = ret.Body.Close() }() + var body []byte + body, err = ioutil.ReadAll(ret.Body) + if err == nil { + if err = json.Unmarshal(body, &resp); err == nil { + return resp, nil + } + } + } + return resp, errors.WithMessage(err, "failed to list PmHosts") +} + +func (c *Client) UpdatePmHost(args *service.PrivateKeyHosts, log *zap.SugaredLogger) error { + args.UpdateStatus = true + body, err := json.Marshal(args) + if err != nil { + log.Errorf("marshal json args error: %v", err) + return err + } + + url := fmt.Sprintf("%s/system/privateKey/%s", c.APIBase, args.ID.Hex()) + request, err := http.NewRequest("PUT", url, bytes.NewBuffer(body)) + if err != nil { + log.Errorf("UpdatePmHost new http request error: %v", err) + return err + } + + var ret *http.Response + if ret, err = c.Conn.Do(request); err == nil { + defer func() { _ = ret.Body.Close() }() + _, err := ioutil.ReadAll(ret.Body) + if err != nil { + return errors.WithMessage(err, "failed to get PmHost") + } + } + + return nil +} diff --git a/pkg/microservice/cron/core/service/client/sonar.go b/pkg/microservice/cron/core/service/client/sonar.go index d97e988430c1973e949b66b43379b15f87a96de0..4b50f102f0dd51bab762f34ce26b724a4ff63a79 100644 --- a/pkg/microservice/cron/core/service/client/sonar.go +++ b/pkg/microservice/cron/core/service/client/sonar.go @@ -24,7 +24,6 @@ import ( "go.uber.org/zap" configbase "github.com/koderover/zadig/pkg/config" - "github.com/koderover/zadig/pkg/setting" ) func (c *Client) InitPullSonarStatScheduler(log *zap.SugaredLogger) error { @@ -45,7 +44,6 @@ func (c *Client) InitPullSonarTestsMeasure(log *zap.SugaredLogger) error { log.Errorf("create post request error : %v", err) return err } - request.Header.Set("Authorization", fmt.Sprintf("%s %s", setting.RootAPIKey, c.Token)) var resp *http.Response resp, err = c.Conn.Do(request) @@ -76,7 +74,6 @@ func (c *Client) InitPullSonarDeliveryMeasure(log *zap.SugaredLogger) error { log.Errorf("create post request error : %v", err) return err } - request.Header.Set("Authorization", fmt.Sprintf("%s %s", setting.RootAPIKey, c.Token)) var resp *http.Response resp, err = c.Conn.Do(request) @@ -107,7 +104,6 @@ func (c *Client) InitPullSonarRepos(log *zap.SugaredLogger) error { log.Errorf("create post request error : %v", err) return err } - request.Header.Set("Authorization", fmt.Sprintf("%s %s", setting.RootAPIKey, c.Token)) var resp *http.Response resp, err = c.Conn.Do(request) diff --git a/pkg/microservice/cron/core/service/client/stat.go b/pkg/microservice/cron/core/service/client/stat.go index 17c3d147129db54c02c1e42a7b704e5d5ab8522f..b341e1653d206262d216b831eeda3151766c1880 100644 --- a/pkg/microservice/cron/core/service/client/stat.go +++ b/pkg/microservice/cron/core/service/client/stat.go @@ -26,23 +26,23 @@ import ( func (c *Client) InitStatData(log *zap.SugaredLogger) error { //build - url := fmt.Sprintf("%s/api/quality/stat/initBuildStat", configbase.AslanxServiceAddress()) + url := fmt.Sprintf("%s/api/stat/quality/initBuildStat", configbase.AslanServiceAddress()) log.Info("start init buildStat..") - err := c.sendRequest(url) + _, err := c.sendPostRequest(url, nil, log) if err != nil { log.Errorf("trigger init buildStat error :%v", err) } //test - url = fmt.Sprintf("%s/api/quality/stat/initTestStat", configbase.AslanxServiceAddress()) + url = fmt.Sprintf("%s/api/stat/quality/initTestStat", configbase.AslanServiceAddress()) log.Info("start init testStat..") - err = c.sendRequest(url) + _, err = c.sendPostRequest(url, nil, log) if err != nil { log.Errorf("trigger init testStat error :%v", err) } //deploy - url = fmt.Sprintf("%s/api/quality/stat/initDeployStat", configbase.AslanxServiceAddress()) + url = fmt.Sprintf("%s/api/stat/quality/initDeployStat", configbase.AslanServiceAddress()) log.Info("start init deployStat..") - err = c.sendRequest(url) + _, err = c.sendPostRequest(url, nil, log) if err != nil { log.Errorf("trigger init deployStat error :%v", err) } diff --git a/pkg/microservice/cron/core/service/client/testing.go b/pkg/microservice/cron/core/service/client/testing.go index bb7e50d9e114541ce96f22cb3da907f57b508169..4154a01086d9f1bc95c39df69e80ca01c86dfb2f 100644 --- a/pkg/microservice/cron/core/service/client/testing.go +++ b/pkg/microservice/cron/core/service/client/testing.go @@ -26,7 +26,6 @@ import ( "go.uber.org/zap" "github.com/koderover/zadig/pkg/microservice/cron/core/service" - "github.com/koderover/zadig/pkg/setting" ) func (c *Client) ListTests(log *zap.SugaredLogger) ([]*service.TestingOpt, error) { @@ -39,7 +38,6 @@ func (c *Client) ListTests(log *zap.SugaredLogger) ([]*service.TestingOpt, error return nil, err } - request.Header.Set("Authorization", fmt.Sprintf("%s %s", setting.RootAPIKey, c.Token)) var ret *http.Response ret, err = c.Conn.Do(request) diff --git a/pkg/microservice/cron/core/service/client/workflow.go b/pkg/microservice/cron/core/service/client/workflow.go index 894a0687a05fab85c565a719a6a5be0074b5e9ba..6884d9f5b341b2e0d25344f7434e7acc596247a3 100644 --- a/pkg/microservice/cron/core/service/client/workflow.go +++ b/pkg/microservice/cron/core/service/client/workflow.go @@ -26,7 +26,6 @@ import ( "go.uber.org/zap" "github.com/koderover/zadig/pkg/microservice/cron/core/service" - "github.com/koderover/zadig/pkg/setting" ) // ListWorkflows ... @@ -71,6 +70,5 @@ func (c *Client) genListWorkFlowReq(log *zap.SugaredLogger) (*http.Request, erro return nil, err } - request.Header.Set(setting.AuthorizationHeader, fmt.Sprintf("%s %s", setting.RootAPIKey, c.Token)) return request, nil } diff --git a/pkg/microservice/cron/core/service/scheduler/cronjob_handler.go b/pkg/microservice/cron/core/service/scheduler/cronjob_handler.go index a3a678fc8e6c277c2c80036d2281b703ff0774ec..a16cd2afa01672e83f3042a64771b05a301f5f74 100644 --- a/pkg/microservice/cron/core/service/scheduler/cronjob_handler.go +++ b/pkg/microservice/cron/core/service/scheduler/cronjob_handler.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "net/http" + "path" "strings" "time" @@ -69,14 +70,10 @@ func InitExistedCronjob(client *client.Client, scheduler *cronlib.CronSchduler) failsafeJobList []*service.Cronjob ) listAPI := fmt.Sprintf("%s/cron/cronjob", client.APIBase) - header := http.Header{} - header.Set("Authorization", fmt.Sprintf("%s %s", setting.TIMERAPIKEY, client.Token)) // failsafe function: get enabled workflow and register them failsafeAPI := fmt.Sprintf("%s/cron/cronjob/failsafe", client.APIBase) cl := httpclient.New( - httpclient.SetAuthScheme(setting.TIMERAPIKEY), - httpclient.SetAuthToken(client.Token), httpclient.SetRetryCount(100), httpclient.SetRetryWaitTime(PullInterval), ) @@ -254,7 +251,7 @@ func (h *CronjobHandler) registerWorkFlowJob(name, schedule string, job *service args.DistributeEnabled = job.WorkflowArgs.DistributeEnabled } scheduleJob, err := cronlib.NewJobModel(schedule, func() { - if err := h.aslanCli.ScheduleCall("workflow/workflowtask", args, log.SugaredLogger()); err != nil { + if err := h.aslanCli.ScheduleCall(path.Join("workflow/workflowtask", args.WorkflowName), args, log.SugaredLogger()); err != nil { log.Errorf("[%s]RunScheduledTask err: %v", name, err) } }) @@ -304,7 +301,6 @@ func (h *CronjobHandler) stopCronjob(name, ptype string) error { var jobList []*service.Cronjob listAPI := fmt.Sprintf("%s/cron/cronjob/type/%s/name/%s", h.aslanCli.APIBase, ptype, name) header := http.Header{} - header.Set("Authorization", fmt.Sprintf("%s %s", setting.TIMERAPIKEY, h.aslanCli.Token)) resp, err := util.SendRequest(listAPI, "GET", header, nil) if err != nil { log.Errorf("Failed to get job list, the error is: %v, reconsuming", err) @@ -359,7 +355,7 @@ func registerCronjob(job *service.Cronjob, client *client.Client, scheduler *cro cron, _ = convertCronString(job.JobType, job.Time, job.Frequency, job.Number) } scheduleJob, err := cronlib.NewJobModel(cron, func() { - if err := client.ScheduleCall("workflow/workflowtask", args, log.SugaredLogger()); err != nil { + if err := client.ScheduleCall(path.Join("workflow/workflowtask", job.WorkflowArgs.WorkflowName), args, log.SugaredLogger()); err != nil { log.Errorf("[%s]RunScheduledTask err: %v", job.Name, err) } }) diff --git a/pkg/microservice/cron/core/service/scheduler/schedule_collie_pipeline.go b/pkg/microservice/cron/core/service/scheduler/schedule_collie_pipeline.go deleted file mode 100644 index ad35b2c8ce59283bb13e4816dfdd67169e0f053a..0000000000000000000000000000000000000000 --- a/pkg/microservice/cron/core/service/scheduler/schedule_collie_pipeline.go +++ /dev/null @@ -1,111 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package scheduler - -import ( - "reflect" - - "github.com/jasonlvhit/gocron" - "go.uber.org/zap" - - "github.com/koderover/zadig/pkg/microservice/cron/core/service" - "github.com/koderover/zadig/pkg/setting" -) - -// UpsertColliePipelineScheduler ... -func (c *CronClient) UpsertColliePipelineScheduler(log *zap.SugaredLogger) { - pipelines, err := c.CollieCli.ListColliePipelines(log) - if err != nil { - log.Error(err) - return - } - - log.Info("start init collie pipeline task scheduler..") - for _, pipeline := range pipelines { - log.Infof("deal collie pipeline data, name:%s, schedule:%v", pipeline.Metadata.Name, pipeline.Spec.Schedules) - key := "collie-pipeline-timer-" + pipeline.Metadata.Name - - schedules := pipeline.Spec.Schedules - // 如果该pipeline先配置了定时器，然后将定时器全部删除了，这时需要把原来的schedule暂停并删除 - if len(schedules) == 0 { - // 暂停 - if _, ok := c.SchedulerController[key]; ok { - c.SchedulerController[key] <- true - delete(c.SchedulerController, key) - } - // 删除 - if _, ok := c.Schedulers[key]; ok { - log.Warnf("delete expired test from Schedulers, key:%s", key) - delete(c.Schedulers, key) - } - if _, ok := c.lastSchedulers[key]; ok { - log.Warnf("delete expired test from lastSchedulers, key:%s", key) - delete(c.lastSchedulers, key) - } - continue - } - - // 判断该测试的定时器配置是否有更新，没有更新则跳过 - if _, ok := c.lastSchedulers[key]; ok && reflect.DeepEqual(schedules, c.lastSchedulers[key]) { - continue - } else { - c.lastSchedulers[key] = schedules - } - - // 定时器有更新，则重新设置该schedule - newScheduler := gocron.NewScheduler() - for _, schedule := range schedules { - if schedule == nil || !schedule.Enabled { - continue - } - log.Infof("deal collie pipeline schedule, name:%s, item:%v", pipeline.Metadata.Name, schedule) - if err := schedule.Validate(); err != nil { - log.Errorf("[%s] invalid schedule: %v", key, err) - continue - } - BuildScheduledJob(newScheduler, schedule).Do(c.RunColliePipelineScheduledTask, pipeline, log) - } - - c.Schedulers[key] = newScheduler - log.Infof("[%s] building schedulers..", key) - // 停掉旧的scheduler - if _, ok := c.SchedulerController[key]; ok { - c.SchedulerController[key] <- true - } - log.Infof("[%s] lens of scheduler: %d", key, c.Schedulers[key].Len()) - c.SchedulerController[key] = c.Schedulers[key].Start() - } -} - -// RunTestScheduledTask ... -func (c *CronClient) RunColliePipelineScheduledTask(pipeline *service.PipelineResource, log *zap.SugaredLogger) { - log.Infof("start collie pipeline cron job: %s ...", pipeline.Metadata.Name) - - args := &service.CreateBuildRequest{ - UserID: setting.CronTaskCreator, - UserName: setting.CronTaskCreator, - NoCache: false, - ResetVolume: false, - Variables: pipeline.Spec.Variables, - PipelineName: pipeline.Metadata.Name, - ProductName: pipeline.Metadata.Project, - } - - if err := c.CollieCli.RunColliePipelineTask(args, log); err != nil { - log.Errorf("[%s]RunTestScheduledTask err: %v", pipeline.Metadata.Name, err) - } -} diff --git a/pkg/microservice/cron/core/service/scheduler/schedule_env.go b/pkg/microservice/cron/core/service/scheduler/schedule_env.go index 8116a5e9cfaccf7ddc1fa8e368d5f955e6b47d25..b25e82cc0ab664b098b724964b782b1fcdcf521e 100644 --- a/pkg/microservice/cron/core/service/scheduler/schedule_env.go +++ b/pkg/microservice/cron/core/service/scheduler/schedule_env.go @@ -33,18 +33,20 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "github.com/koderover/zadig/pkg/microservice/cron/core/service" + "github.com/koderover/zadig/pkg/microservice/cron/core/service/client" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/types" ) // UpsertEnvServiceScheduler ... func (c *CronClient) UpsertEnvServiceScheduler(log *zap.SugaredLogger) { - envs, err := c.AslanCli.ListEnvs(log) + envs, err := c.AslanCli.ListEnvs(log, &client.EvnListOption{BasicFacility: setting.BasicFacilityCVM}) if err != nil { log.Error(err) return } //当前的环境数据和上次做比较，如果环境有删除或者环境中的服务有删除，要清理掉定时器 - c.compareProductRevision(envs, log) + c.comparePMProductRevision(envs, log) log.Info("start init env scheduler..") taskMap := make(map[string]bool) @@ -80,33 +82,28 @@ func (c *CronClient) UpsertEnvServiceScheduler(log *zap.SugaredLogger) { } continue } - for _, envConfig := range svc.EnvConfigs { - if envConfig.EnvName != env.EnvName { + for _, envStatus := range svc.EnvStatuses { + if envStatus.EnvName != env.EnvName { continue } - for _, hostID := range envConfig.HostIDs { - for _, healthCheck := range svc.HealthChecks { - key := "service-" + serviceRevision.ServiceName + "-" + setting.PMDeployType + "-" + - env.EnvName + "-" + hostID + "-" + healthCheck.Protocol + "-" + strconv.Itoa(healthCheck.Port) + "-" + healthCheck.Path - taskMap[key] = true - if _, ok := c.lastServiceSchedulers[key]; ok && reflect.DeepEqual(serviceRevision, c.lastServiceSchedulers[key]) { - continue - } else { - c.lastServiceSchedulers[key] = serviceRevision - } - hostInfo, _ := c.AslanCli.GetHostInfo(hostID, log) - if hostInfo == nil { - continue - } - newScheduler := gocron.NewScheduler() - BuildScheduledEnvJob(newScheduler, healthCheck).Do(c.RunScheduledService, svc, healthCheck, hostInfo.IP, env.EnvName, hostID, log) - c.Schedulers[key] = newScheduler - log.Infof("[%s] service schedulers..", key) - c.Schedulers[key].Start() + + for _, healthCheck := range svc.HealthChecks { + key := "service-" + serviceRevision.ServiceName + "-" + setting.PMDeployType + "-" + + env.EnvName + "-" + envStatus.HostID + "-" + healthCheck.Protocol + "-" + strconv.Itoa(healthCheck.Port) + "-" + healthCheck.Path + taskMap[key] = true + if _, ok := c.lastServiceSchedulers[key]; ok && reflect.DeepEqual(serviceRevision, c.lastServiceSchedulers[key]) { + continue } + c.lastServiceSchedulers[key] = serviceRevision + + newScheduler := gocron.NewScheduler() + BuildScheduledEnvJob(newScheduler, healthCheck).Do(c.RunScheduledService, svc, healthCheck, envStatus.Address, env.EnvName, envStatus.HostID, log) + c.Schedulers[key] = newScheduler + log.Infof("[%s] service schedulers..", key) + c.Schedulers[key].Start() } - break } + break } } } @@ -200,6 +197,7 @@ func BuildScheduledEnvJob(scheduler *gocron.Scheduler, healthCheck *service.PmHe } return scheduler.Every(interval).Seconds() } + func runProbe(healthCheck *service.PmHealthCheck, address string, log *zap.SugaredLogger) (string, error) { var ( message string @@ -208,7 +206,7 @@ func runProbe(healthCheck *service.PmHealthCheck, address string, log *zap.Sugar timeout := time.Duration(healthCheck.TimeOut) * time.Second switch healthCheck.Protocol { case setting.ProtocolHTTP, setting.ProtocolHTTPS: - if message, err = doHTTPProbe(healthCheck.Protocol, address, healthCheck.Path, healthCheck.Port, timeout, log); err != nil { + if message, err = doHTTPProbe(healthCheck.Protocol, address, healthCheck.Path, healthCheck.Port, []*types.HTTPHeader{}, timeout, "", log); err != nil { log.Errorf("doHttpProbe err:%v", err) return Failure, err } @@ -243,7 +241,7 @@ func doTCPProbe(addr string, port int, timeout time.Duration, log *zap.SugaredLo return Success, nil } -func doHTTPProbe(protocol, address, path string, port int, timeout time.Duration, log *zap.SugaredLogger) (string, error) { +func doHTTPProbe(protocol, address, path string, port int, headerList []*types.HTTPHeader, timeout time.Duration, responseSuccessFlag string, log *zap.SugaredLogger) (string, error) { tlsConfig := &tls.Config{InsecureSkipVerify: true} transport := &http.Transport{ TLSClientConfig: tlsConfig, @@ -259,10 +257,14 @@ func doHTTPProbe(protocol, address, path string, port int, timeout time.Duration if err != nil { return Failure, err } + headers := buildHeader(headerList) req, err := http.NewRequest("GET", url, nil) if err != nil { return Failure, err } + req.Header = headers + req.Host = headers.Get("Host") + res, err := client.Do(req) if err != nil { return Failure, err @@ -274,10 +276,13 @@ func doHTTPProbe(protocol, address, path string, port int, timeout time.Duration } if res.StatusCode >= http.StatusOK && res.StatusCode < http.StatusBadRequest { + if responseSuccessFlag != "" && !strings.Contains(string(body), responseSuccessFlag) { + return Failure, fmt.Errorf("HTTP probe failed with response success flag: %s", responseSuccessFlag) + } log.Infof("Probe succeeded for %s, Response: %v", url, *res) return Success, nil } - log.Infof("Probe failed for %s, response body: %v", url, string(body)) + log.Warnf("Probe failed for %s, response body: %v", url, string(body)) return Failure, fmt.Errorf("HTTP probe failed with statuscode: %d", res.StatusCode) } @@ -314,28 +319,67 @@ func formatURL(protocol, address, path string, port int) (string, error) { return fmt.Sprintf("%s://%s:%d/%s", protocol, address, port, path), nil } -// 先比较环境，再比较服务 -func (c *CronClient) compareProductRevision(currentProductRevisions []*service.ProductRevision, log *zap.SugaredLogger) { - if len(c.lastProductRevisions) == 0 { - c.lastProductRevisions = currentProductRevisions +func buildHeader(headerList []*types.HTTPHeader) http.Header { + headers := make(http.Header) + for _, header := range headerList { + headers[header.Name] = append(headers[header.Name], header.Value) + } + return headers +} + +func buildEnvNameKey(productRevision *service.ProductRevision) string { + return "helm-values-sync-" + productRevision.ProductName + "-" + productRevision.EnvName +} + +func (c *CronClient) compareHelmProductEnvRevision(currentProductRevisions []*service.ProductRevision, log *zap.SugaredLogger) { + if len(c.lastHelmProductRevisions) == 0 { + c.lastHelmProductRevisions = currentProductRevisions + return + } + deleteProductRevisions := make([]*service.ProductRevision, 0) + curEnvSet := sets.NewString() + for _, r := range currentProductRevisions { + curEnvSet.Insert(buildEnvNameKey(r)) + } + for _, lastProductRevision := range c.lastHelmProductRevisions { + if !curEnvSet.Has(buildEnvNameKey(lastProductRevision)) { + deleteProductRevisions = append(deleteProductRevisions, lastProductRevision) + } + } + // delete related schedulers when env is deleted + for _, env := range deleteProductRevisions { + envKey := buildEnvNameKey(env) + if _, ok := c.SchedulerController[envKey]; ok { + c.SchedulerController[envKey] <- true + } + if _, ok := c.Schedulers[envKey]; ok { + c.Schedulers[envKey].Clear() + delete(c.Schedulers, envKey) + } + } + c.lastHelmProductRevisions = currentProductRevisions +} + +// compare environments and then services +func (c *CronClient) comparePMProductRevision(currentProductRevisions []*service.ProductRevision, log *zap.SugaredLogger) { + if len(c.lastPMProductRevisions) == 0 { + c.lastPMProductRevisions = currentProductRevisions return } deleteProductRevisions := make([]*service.ProductRevision, 0) lastProductSvcRevisionMap := make(map[string][]*service.SvcRevision) currentProductSvcRevisionMap := make(map[string][]*service.SvcRevision) - for _, lastProductRevision := range c.lastProductRevisions { + for _, lastProductRevision := range c.lastPMProductRevisions { isContain := false for _, currentProductRevision := range currentProductRevisions { if currentProductRevision.ProductName == lastProductRevision.ProductName && currentProductRevision.EnvName == lastProductRevision.EnvName { - currentProductSvcRevisionMap[currentProductRevision.ProductName+"-"+currentProductRevision.EnvName] = currentProductRevision.ServiceRevisions lastProductSvcRevisionMap[lastProductRevision.ProductName+"-"+lastProductRevision.EnvName] = lastProductRevision.ServiceRevisions isContain = true break } } - if !isContain { deleteProductRevisions = append(deleteProductRevisions, lastProductRevision) } @@ -456,7 +500,7 @@ func (c *CronClient) compareProductRevision(currentProductRevisions []*service.P } } - c.lastProductRevisions = currentProductRevisions + c.lastPMProductRevisions = currentProductRevisions } const ( diff --git a/pkg/microservice/cron/core/service/scheduler/schedule_env_update.go b/pkg/microservice/cron/core/service/scheduler/schedule_env_update.go new file mode 100644 index 0000000000000000000000000000000000000000..f9601e1024e56eee6289e3c8dce69d1f6471a6ef --- /dev/null +++ b/pkg/microservice/cron/core/service/scheduler/schedule_env_update.go @@ -0,0 +1,105 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scheduler + +import ( + "github.com/jasonlvhit/gocron" + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/cron/core/service" + "github.com/koderover/zadig/pkg/microservice/cron/core/service/client" + "github.com/koderover/zadig/pkg/setting" +) + +// EnvUpdateInterval TODO interval will be set on product in future +const EnvUpdateInterval = 30 + +func needCreateSchedule(rendersetObj *service.ProductRenderset) bool { + if rendersetObj.YamlData != nil && rendersetObj.YamlData.Source == setting.SourceFromGitRepo { + return true + } + for _, chartData := range rendersetObj.ChartInfos { + if chartData.OverrideYaml != nil && chartData.OverrideYaml.Source == setting.SourceFromGitRepo { + return true + } + } + return false +} + +func (c *CronClient) UpsertEnvValueSyncScheduler(log *zap.SugaredLogger) { + envs, err := c.AslanCli.ListEnvs(log, &client.EvnListOption{DeployType: []string{setting.HelmDeployType}}) + if err != nil { + log.Error(err) + return + } + + //compare to last revision, delete related schedulers when env is deleted + c.compareHelmProductEnvRevision(envs, log) + + log.Info("start init env values sync scheduler..") + for _, env := range envs { + envObj, err := c.AslanCli.GetEnvService(env.ProductName, env.EnvName, log) + if err != nil { + log.Errorf("failed to get env data, productName:%s envName:%s err:%v", env.ProductName, env.EnvName, err) + continue + } + if envObj.Render == nil { + log.Errorf("render info of product: %s:%s is nil", env.ProductName, env.EnvName) + continue + } + + rendersetObj, err := c.AslanCli.GetRenderset(envObj.Render.Name, envObj.Render.Revision, log) + if err != nil { + log.Errorf("failed to get renderset data: %s, err:%s", envObj.Render.Name, err) + continue + } + + envKey := buildEnvNameKey(env) + if lastEnvData, ok := c.lastEnvSchedulerData[envKey]; ok { + // render not changed, no need to update scheduler + if lastEnvData.Render.Revision == rendersetObj.Revision { + continue + } + } + c.lastEnvSchedulerData[envKey] = envObj + if _, ok := c.SchedulerController[envKey]; ok { + c.SchedulerController[envKey] <- true + } + if _, ok := c.Schedulers[envKey]; ok { + c.Schedulers[envKey].Clear() + delete(c.Schedulers, envKey) + } + + if !needCreateSchedule(rendersetObj) { + continue + } + + newScheduler := gocron.NewScheduler() + newScheduler.Every(EnvUpdateInterval).Seconds().Do(c.RunScheduledEnvUpdate, env.ProductName, env.EnvName, log) + c.Schedulers[envKey] = newScheduler + log.Infof("[%s] add schedulers..", envKey) + c.SchedulerController[envKey] = c.Schedulers[envKey].Start() + } +} + +func (c *CronClient) RunScheduledEnvUpdate(productName, envName string, log *zap.SugaredLogger) { + log.Infof("start to Run ScheduledEnvUpdate, productName: %s, envName: %s", productName, envName) + err := c.AslanCli.SyncEnvVariables(productName, envName, log) + if err != nil { + log.Errorf("failed to sync variables for env: %s:%s", productName, envName) + } +} diff --git a/pkg/microservice/cron/core/service/scheduler/schedule_pm.go b/pkg/microservice/cron/core/service/scheduler/schedule_pm.go new file mode 100644 index 0000000000000000000000000000000000000000..3c090a80398869cdd2aef1318372695608854eb9 --- /dev/null +++ b/pkg/microservice/cron/core/service/scheduler/schedule_pm.go @@ -0,0 +1,79 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scheduler + +import ( + "time" + + "github.com/koderover/zadig/pkg/microservice/cron/core/service" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/types" + "go.uber.org/zap" +) + +func (c *CronClient) UpdatePmHostStatusScheduler(log *zap.SugaredLogger) { + hosts, err := c.AslanCli.ListPmHosts(log) + if err != nil { + log.Error(err) + return + } + + log.Info("start init pm host status scheduler..") + for _, hostElem := range hosts { + go func(hostPm *service.PrivateKeyHosts, log *zap.SugaredLogger) { + if hostPm.Port == 0 { + hostPm.Port = setting.PMHostDefaultPort + } + newStatus := setting.PMHostStatusAbnormal + if hostPm.Probe == nil { + hostPm.Probe = &types.Probe{ProbeScheme: setting.ProtocolTCP} + } + var err error + msg := "" + + switch hostPm.Probe.ProbeScheme { + case setting.ProtocolHTTP, setting.ProtocolHTTPS: + if hostPm.Probe.HttpProbe == nil { + break + } + msg, err = doHTTPProbe(string(hostPm.Probe.ProbeScheme), hostPm.IP, hostPm.Probe.HttpProbe.Path, hostPm.Probe.HttpProbe.Port, hostPm.Probe.HttpProbe.HTTPHeaders, time.Duration(hostPm.Probe.HttpProbe.TimeOutSecond)*time.Second, hostPm.Probe.HttpProbe.ResponseSuccessFlag, log) + if err != nil { + log.Warnf("doHttpProbe err:%s", err) + } + case setting.ProtocolTCP: + msg, err = doTCPProbe(hostPm.IP, int(hostPm.Port), 3*time.Second, log) + if err != nil { + log.Warnf("doTCPProbe TCP %s:%d err: %s)", hostPm.IP, hostPm.Port, err) + } + } + + if msg == Success { + newStatus = setting.PMHostStatusNormal + } + + if hostPm.Status == newStatus { + return + } + hostPm.Status = newStatus + + err = c.AslanCli.UpdatePmHost(hostPm, log) + if err != nil { + log.Error(err) + } + }(hostElem, log) + } +} diff --git a/pkg/microservice/cron/core/service/scheduler/schedule_workflow.go b/pkg/microservice/cron/core/service/scheduler/schedule_workflow.go index a3444de724cd692319cacd452ac70bf8ecc53a72..54e4407759e7a4cdc63185ae17c8d3e97e0ee534 100644 --- a/pkg/microservice/cron/core/service/scheduler/schedule_workflow.go +++ b/pkg/microservice/cron/core/service/scheduler/schedule_workflow.go @@ -123,23 +123,32 @@ func (c *CronClient) UpsertWorkflowScheduler(log *zap.SugaredLogger) { ScheduleNames := sets.NewString( CleanJobScheduler, UpsertWorkflowScheduler, UpsertTestScheduler, InitStatScheduler, InitOperationStatScheduler, - CleanProductScheduler, InitHealthCheckScheduler, UpsertColliePipelineScheduler) + CleanProductScheduler, InitHealthCheckScheduler, InitHealthCheckPmHostScheduler, + UpsertColliePipelineScheduler, InitHelmEnvSyncValuesScheduler, EnvResourceSyncScheduler) // 停掉已被删除的pipeline对应的scheduler for name := range c.Schedulers { if _, ok := taskMap[name]; !ok && !ScheduleNames.Has(name) { - //排除非容器部署服务健康检查定时器 + // exclude service heath check timers if strings.HasPrefix(name, "service-") && strings.Contains(name, "pm") { continue } - // 排除测试定时任务定时器 + // exclude test timers if strings.HasPrefix(name, "test-timer-") { continue } - // 排除自由编排工作流定时器 + // exclude collie pipeline timers if strings.HasPrefix(name, "collie-pipeline-timer-") { continue } + // exclude helm env values sync timers + if strings.HasPrefix(name, "helm-values-sync-") { + continue + } + // exclude env resource sync timers + if strings.HasPrefix(name, "env-resource") { + continue + } log.Warnf("[%s]deleted workflow detached", name) if _, ok := c.SchedulerController[name]; ok { c.SchedulerController[name] <- true diff --git a/pkg/microservice/cron/core/service/scheduler/scheduler.go b/pkg/microservice/cron/core/service/scheduler/scheduler.go index 5ab7e04642e3b363c6bc316608179b439607348f..9b5237cd16a3a5e815a4b1b685546444a5a65b58 100644 --- a/pkg/microservice/cron/core/service/scheduler/scheduler.go +++ b/pkg/microservice/cron/core/service/scheduler/scheduler.go @@ -20,7 +20,7 @@ import ( "fmt" stdlog "log" "os" - "strings" + "reflect" "time" "github.com/jasonlvhit/gocron" @@ -28,54 +28,111 @@ import ( "github.com/rfyiamcool/cronlib" "go.uber.org/zap" + newgoCron "github.com/go-co-op/gocron" + configbase "github.com/koderover/zadig/pkg/config" "github.com/koderover/zadig/pkg/microservice/cron/config" "github.com/koderover/zadig/pkg/microservice/cron/core/service" "github.com/koderover/zadig/pkg/microservice/cron/core/service/client" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/poetry" + "github.com/koderover/zadig/pkg/shared/client/aslan" "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types" ) // CronClient ... type CronClient struct { - AslanCli *client.Client - CollieCli *client.CollieClient - Schedulers map[string]*gocron.Scheduler - SchedulerController map[string]chan bool - lastSchedulers map[string][]*service.Schedule - lastServiceSchedulers map[string]*service.SvcRevision - enabledMap map[string]bool - lastProductRevisions []*service.ProductRevision - log *zap.SugaredLogger + AslanCli *client.Client + Schedulers map[string]*gocron.Scheduler + SchedulerController map[string]chan bool + lastSchedulers map[string][]*service.Schedule + lastServiceSchedulers map[string]*service.SvcRevision + lastEnvSchedulerData map[string]*service.ProductResp + lastEnvResourceSchedulerData map[string]*service.EnvResource + enabledMap map[string]bool + lastPMProductRevisions []*service.ProductRevision + lastHelmProductRevisions []*service.ProductRevision + log *zap.SugaredLogger +} + +type CronV3Client struct { + Scheduler *newgoCron.Scheduler + AslanCli *aslan.Client +} + +func NewCronV3() *CronV3Client { + return &CronV3Client{ + Scheduler: newgoCron.NewScheduler(time.Local), + AslanCli: aslan.New(configbase.AslanServiceAddress()), + } +} + +func (c *CronV3Client) Start() { + var lastConfig *aslan.CleanConfig + + c.Scheduler.Every(5).Seconds().Do(func() { + // get the docker clean config + config, err := c.AslanCli.GetDockerCleanConfig() + if err != nil { + log.Errorf("get config err :%s", err) + return + } + + if !reflect.DeepEqual(lastConfig, config) { + lastConfig = config + log.Infof("config changed to %v", config) + if config.CronEnabled { + c.Scheduler.RemoveByTag(string(types.CleanDockerTag)) + _, err = c.Scheduler.Cron(config.Cron).Tag(string(types.CleanDockerTag)).Do(func() { + log.Infof("trigger aslan docker clean,reg: %v", config.Cron) + // call docker clean + if err := c.AslanCli.DockerClean(); err != nil { + log.Errorf("fail to clean docker cache , err:%s", err) + } + }) + if err != nil { + log.Errorf("fail to add docker_cache clean cron job:reg: %v,err:%s", config.Cron, err) + } + } else { + log.Infof("remove docker_cache clean job , job tag: %v", types.CleanDockerTag) + c.Scheduler.RemoveByTag(string(types.CleanDockerTag)) + + } + } + }) + + c.Scheduler.StartAsync() } const ( - // CleanJobScheduler ... CleanJobScheduler = "CleanJobScheduler" - // UpsertWorkflowScheduler ... + UpsertWorkflowScheduler = "UpsertWorkflowScheduler" - // UpsertTestScheduler ... + UpsertTestScheduler = "UpsertTestScheduler" - // UpsertColliePipelineScheduler ... + UpsertColliePipelineScheduler = "UpsertColliePipelineScheduler" - //CleanProductScheduler ... + CleanProductScheduler = "CleanProductScheduler" - //InitBuildStatScheduler + + CleanCIResourcesScheduler = "CleanCIResourcesScheduler" + InitStatScheduler = "InitStatScheduler" - //InitOperationStatScheduler + InitOperationStatScheduler = "InitOperationStatScheduler" - //InitPullSonarStatScheduler InitPullSonarStatScheduler = "InitPullSonarStatScheduler" // SystemCapacityGC periodically triggers garbage collection for system data based on its retention policy. SystemCapacityGC = "SystemCapacityGC" - //InitHealthCheckScheduler + InitHealthCheckScheduler = "InitHealthCheckScheduler" - // FreestyleType 自由编排工作流 - freestyleType = "freestyle" + InitHealthCheckPmHostScheduler = "InitHealthCheckPmHostScheduler" + + InitHelmEnvSyncValuesScheduler = "InitHelmEnvSyncValuesScheduler" + + EnvResourceSyncScheduler = "EnvResourceSyncScheduler" ) // NewCronClient ... @@ -83,8 +140,7 @@ const ( func NewCronClient() *CronClient { nsqLookupAddrs := config.NsqLookupAddrs() - aslanCli := client.NewAslanClient(fmt.Sprintf("%s/api", configbase.AslanServiceAddress()), config.RootToken()) - collieCli := client.NewCollieClient(config.CollieAPI(), config.CollieToken()) + aslanCli := client.NewAslanClient(fmt.Sprintf("%s/api", configbase.AslanServiceAddress())) //初始化nsq config := nsq.NewConfig() // 注意 WD_POD_NAME 必须使用 Downward API 配置环境变量 @@ -92,14 +148,6 @@ func NewCronClient() *CronClient { config.MaxAttempts = 50 config.LookupdPollInterval = 1 * time.Second - //nsqClient := nsqcli.NewNsqClient(nsqLookupAddrs, "127.0.0.1:4151") - //// 初始化nsq topic - //err := nsqClient.EnsureNsqdTopics([]string{setting.TopicAck, setting.TopicItReport, setting.TopicNotification}) - //if err != nil { - // //FIXME - // log.Fatalf("cannot ensure nsq topic, the error is %v", err) - //} - //Cronjob Client cronjobClient, err := nsq.NewConsumer(setting.TopicCronjob, "cronjob", config) if err != nil { @@ -119,14 +167,15 @@ func NewCronClient() *CronClient { } return &CronClient{ - AslanCli: aslanCli, - CollieCli: collieCli, - Schedulers: make(map[string]*gocron.Scheduler), - lastSchedulers: make(map[string][]*service.Schedule), - lastServiceSchedulers: make(map[string]*service.SvcRevision), - SchedulerController: make(map[string]chan bool), - enabledMap: make(map[string]bool), - log: log.SugaredLogger(), + AslanCli: aslanCli, + Schedulers: make(map[string]*gocron.Scheduler), + lastSchedulers: make(map[string][]*service.Schedule), + lastServiceSchedulers: make(map[string]*service.SvcRevision), + lastEnvSchedulerData: make(map[string]*service.ProductResp), + lastEnvResourceSchedulerData: make(map[string]*service.EnvResource), + SchedulerController: make(map[string]chan bool), + enabledMap: make(map[string]bool), + log: log.SugaredLogger(), } } @@ -141,14 +190,10 @@ func (c *CronClient) Init() { // 测试管理的定时任务触发 c.InitTestScheduler() - // 自由编排工作流定时任务触发 - features, _ := getFeatures() - if strings.Contains(features, freestyleType) { - c.InitColliePipelineScheduler() - } - // 定时清理环境 c.InitCleanProductScheduler() + // clean collaboration instance resource every 5 minutes + c.InitCleanCIResourcesScheduler() // 定时初始化构建数据 c.InitBuildStatScheduler() // 定时器初始化话运营统计数据 @@ -157,19 +202,14 @@ func (c *CronClient) Init() { c.InitPullSonarStatScheduler() // 定时初始化健康检查 c.InitHealthCheckScheduler() + // Timing probe host status + c.InitHealthCheckPmHostScheduler() + // sync values from remote for helm envs at regular intervals + c.InitHelmEnvSyncValuesScheduler() + // sync env resources from git at regular intervals + c.InitEnvResourceSyncScheduler() } -func getFeatures() (string, error) { - cl := poetry.New(configbase.PoetryServiceAddress(), config.RootToken()) - fs, err := cl.ListFeatures() - if err != nil { - return "", err - } - - return strings.Join(fs, ","), nil -} - -// InitCleanJobScheduler ... func (c *CronClient) InitCleanJobScheduler() { c.Schedulers[CleanJobScheduler] = gocron.NewScheduler() @@ -179,7 +219,6 @@ func (c *CronClient) InitCleanJobScheduler() { c.Schedulers[CleanJobScheduler].Start() } -// InitCleanProductScheduler ... func (c *CronClient) InitCleanProductScheduler() { c.Schedulers[CleanProductScheduler] = gocron.NewScheduler() @@ -189,7 +228,15 @@ func (c *CronClient) InitCleanProductScheduler() { c.Schedulers[CleanProductScheduler].Start() } -// InitJobScheduler ... +func (c *CronClient) InitCleanCIResourcesScheduler() { + + c.Schedulers[CleanCIResourcesScheduler] = gocron.NewScheduler() + + c.Schedulers[CleanCIResourcesScheduler].Every(5).Minutes().Do(c.AslanCli.TriggerCleanCIResources, c.log) + + c.Schedulers[CleanCIResourcesScheduler].Start() +} + func (c *CronClient) InitJobScheduler() { c.Schedulers[UpsertWorkflowScheduler] = gocron.NewScheduler() @@ -199,7 +246,6 @@ func (c *CronClient) InitJobScheduler() { c.Schedulers[UpsertWorkflowScheduler].Start() } -// InitTestScheduler ... func (c *CronClient) InitTestScheduler() { c.Schedulers[UpsertTestScheduler] = gocron.NewScheduler() @@ -209,19 +255,7 @@ func (c *CronClient) InitTestScheduler() { c.Schedulers[UpsertTestScheduler].Start() } -// InitJobScheduler ... -func (c *CronClient) InitColliePipelineScheduler() { - - c.Schedulers[UpsertColliePipelineScheduler] = gocron.NewScheduler() - - c.Schedulers[UpsertColliePipelineScheduler].Every(1).Minutes().Do(c.UpsertColliePipelineScheduler, c.log) - - c.Schedulers[UpsertColliePipelineScheduler].Start() -} - -// InitBuildStatScheduler ... func (c *CronClient) InitBuildStatScheduler() { - c.Schedulers[InitStatScheduler] = gocron.NewScheduler() c.Schedulers[InitStatScheduler].Every(1).Day().At("01:00").Do(c.AslanCli.InitStatData, c.log) @@ -229,7 +263,6 @@ func (c *CronClient) InitBuildStatScheduler() { c.Schedulers[InitStatScheduler].Start() } -// InitOperationStatScheduler ... func (c *CronClient) InitOperationStatScheduler() { c.Schedulers[InitOperationStatScheduler] = gocron.NewScheduler() @@ -239,7 +272,6 @@ func (c *CronClient) InitOperationStatScheduler() { c.Schedulers[InitOperationStatScheduler].Start() } -// InitPullSonarStatScheduler ... func (c *CronClient) InitPullSonarStatScheduler() { c.Schedulers[InitPullSonarStatScheduler] = gocron.NewScheduler() @@ -266,3 +298,29 @@ func (c *CronClient) InitHealthCheckScheduler() { c.Schedulers[InitHealthCheckScheduler].Start() } + +func (c *CronClient) InitHealthCheckPmHostScheduler() { + + c.Schedulers[InitHealthCheckPmHostScheduler] = gocron.NewScheduler() + + c.Schedulers[InitHealthCheckPmHostScheduler].Every(10).Seconds().Do(c.UpdatePmHostStatusScheduler, c.log) + + c.Schedulers[InitHealthCheckPmHostScheduler].Start() +} + +func (c *CronClient) InitHelmEnvSyncValuesScheduler() { + + c.Schedulers[InitHelmEnvSyncValuesScheduler] = gocron.NewScheduler() + + c.Schedulers[InitHelmEnvSyncValuesScheduler].Every(20).Seconds().Do(c.UpsertEnvValueSyncScheduler, c.log) + + c.Schedulers[InitHelmEnvSyncValuesScheduler].Start() +} + +func (c *CronClient) InitEnvResourceSyncScheduler() { + c.Schedulers[EnvResourceSyncScheduler] = gocron.NewScheduler() + + c.Schedulers[EnvResourceSyncScheduler].Every(20).Seconds().Do(c.UpsertEnvResourceSyncScheduler, c.log) + + c.Schedulers[EnvResourceSyncScheduler].Start() +} diff --git a/pkg/microservice/cron/core/service/scheduler/shedule_env_resource_update.go b/pkg/microservice/cron/core/service/scheduler/shedule_env_resource_update.go new file mode 100644 index 0000000000000000000000000000000000000000..e217daee51fa7080a3094a40061580cb009eddc2 --- /dev/null +++ b/pkg/microservice/cron/core/service/scheduler/shedule_env_resource_update.go @@ -0,0 +1,106 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scheduler + +import ( + "fmt" + + "github.com/jasonlvhit/gocron" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/koderover/zadig/pkg/microservice/cron/core/service" + "github.com/koderover/zadig/pkg/microservice/cron/core/service/client" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/log" +) + +func buildEnvResourceCronKey(envResource *service.EnvResource) string { + return fmt.Sprintf("env-resource:%s:%s-type:%s-name:%s", envResource.ProductName, envResource.EnvName, envResource.Type, envResource.Name) +} + +func (c *CronClient) deleteEnvResourceScheduler(envResourceKey string) { + log.Infof("deleting single env resource scheduler: %s", envResourceKey) + if _, ok := c.SchedulerController[envResourceKey]; ok { + c.SchedulerController[envResourceKey] <- true + delete(c.SchedulerController, envResourceKey) + } + if _, ok := c.Schedulers[envResourceKey]; ok { + c.Schedulers[envResourceKey].Clear() + delete(c.Schedulers, envResourceKey) + } +} + +func (c *CronClient) UpsertEnvResourceSyncScheduler(log *zap.SugaredLogger) { + + envs, err := c.AslanCli.ListEnvs(log, &client.EvnListOption{DeployType: []string{setting.HelmDeployType, setting.K8SDeployType}}) + if err != nil { + log.Errorf("failed to list envs for env resource update: %s", err) + return + } + + log.Info("start init env resource sync scheduler.") + + lastScheduler := sets.NewString() + for k := range c.lastEnvResourceSchedulerData { + lastScheduler.Insert(k) + } + + for _, env := range envs { + envResources, err := c.AslanCli.ListEnvResources(env.ProductName, env.EnvName, log) + if err != nil { + log.Error(err) + return + } + + for _, envResource := range envResources { + envResourceKey := buildEnvResourceCronKey(envResource) + + lastScheduler.Delete(envResourceKey) + + if lastEnvResConfig, ok := c.lastEnvResourceSchedulerData[envResourceKey]; ok { + if envResource.CreateTime == lastEnvResConfig.CreateTime { + continue + } + } + c.lastEnvResourceSchedulerData[envResourceKey] = envResource + + c.deleteEnvResourceScheduler(envResourceKey) + + newScheduler := gocron.NewScheduler() + newScheduler.Every(EnvUpdateInterval).Seconds().Do(c.RunScheduledEnvResourceUpdate, envResource.ProductName, envResource.EnvName, envResource.Type, envResource.Name, log) + + log.Infof("[%s] add env resource schedulers..", envResourceKey) + c.Schedulers[envResourceKey] = newScheduler + c.SchedulerController[envResourceKey] = c.Schedulers[envResourceKey].Start() + } + + } + + for _, k := range lastScheduler.List() { + c.deleteEnvResourceScheduler(k) + delete(c.lastEnvResourceSchedulerData, k) + } +} + +func (c *CronClient) RunScheduledEnvResourceUpdate(productName, envName, resType, resName string, log *zap.SugaredLogger) { + log.Infof("start to Run RunScheduledEnvResourceUpdate, productName: %s, envName: %s, resType: %s, resName: %s", productName, envName, resType, resName) + err := c.AslanCli.SyncEnvResource(productName, envName, resType, resName, log) + if err != nil { + log.Warnf("failed to sync variables for env: %s:%s", productName, envName) + } +} diff --git a/pkg/microservice/cron/core/service/service.go b/pkg/microservice/cron/core/service/service.go index b146140f30ec21cc22c31f029463554eec46903a..fda848aaac25a5ff120dcbdf0bf6cd9be3795fcd 100644 --- a/pkg/microservice/cron/core/service/service.go +++ b/pkg/microservice/cron/core/service/service.go @@ -16,6 +16,14 @@ limitations under the License. package service +import ( + "go.mongodb.org/mongo-driver/bson/primitive" + + templatemodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/types" +) + type SvcRevision struct { ServiceName string `json:"service_name"` Type string `json:"type"` @@ -53,6 +61,20 @@ type ProductRevision struct { IsPublic bool `json:"isPublic"` } +type EnvResource struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + Type string `bson:"type" json:"type"` + ProductName string `bson:"product_name" json:"product_name"` + CreateTime int64 `bson:"create_time" json:"create_time"` + UpdateUserName string `bson:"update_user_name" json:"update_user_name"` + DeletedAt int64 `bson:"deleted_at" json:"deleted_at" ` + Namespace string `bson:"namespace,omitempty" json:"namespace,omitempty"` + EnvName string `bson:"env_name" json:"env_name"` + Name string `bson:"name" json:"name"` + YamlData string `bson:"yaml_data" json:"yaml_data"` + AutoSync bool `bson:"auto_sync" json:"auto_sync"` +} + type ProductResp struct { ID string `json:"id"` ProductName string `json:"product_name"` @@ -72,9 +94,19 @@ type ProductResp struct { Source string `json:"source"` } +type ProductRenderset struct { + Name string `bson:"name" json:"name"` + Revision int64 `bson:"revision" json:"revision"` + EnvName string `bson:"env_name,omitempty" json:"env_name,omitempty"` + ProductTmpl string `bson:"product_tmpl" json:"product_tmpl"` + YamlData *templatemodels.CustomYaml `bson:"yaml_data,omitempty" json:"yaml_data,omitempty"` + ChartInfos []*templatemodels.RenderChart `bson:"chart_infos,omitempty" json:"chart_infos,omitempty"` +} + type EnvConfig struct { EnvName string `json:"env_name"` HostIDs []string `json:"host_ids"` + Labels []string `json:"labels"` } type Service struct { @@ -98,6 +130,15 @@ type PmHealthCheck struct { CurrentUnhealthyNum int `json:"current_unhealthy_num,omitempty"` } +type PrivateKeyHosts struct { + ID primitive.ObjectID `json:"id,omitempty"` + IP string `json:"ip"` + Port int64 `json:"port"` + Status setting.PMHostStatus `json:"status"` + Probe *types.Probe `json:"probe"` + UpdateStatus bool `json:"update_status"` +} + type EnvStatus struct { HostID string `json:"host_id"` EnvName string `json:"env_name"` diff --git a/pkg/microservice/cron/core/service/testing.go b/pkg/microservice/cron/core/service/testing.go index 9dc549fa055bf4e25970098979adaa8d491d58aa..58fa1193a006459611796c12c03fe2fc213280f2 100644 --- a/pkg/microservice/cron/core/service/testing.go +++ b/pkg/microservice/cron/core/service/testing.go @@ -26,6 +26,5 @@ type TestingOpt struct { ExecuteNum int `json:"execute_num,omitempty"` PassRate float64 `json:"pass_rate,omitempty"` AvgDuration float64 `json:"avg_duration,omitempty"` - Workflows []*Workflow `json:"workflows,omitempty"` Schedules *ScheduleCtrl `json:"schedules,omitempty"` } diff --git a/pkg/microservice/cron/core/service/types.go b/pkg/microservice/cron/core/service/types.go index 2fdddf7cebad7e5d7092e9101ddc19b9285b5746..60923cf2ca5222e072e61aa0486d9574c0202689 100644 --- a/pkg/microservice/cron/core/service/types.go +++ b/pkg/microservice/cron/core/service/types.go @@ -24,6 +24,7 @@ import ( "github.com/koderover/zadig/pkg/setting" e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/types" ) // ScheduleType 触发模式 @@ -260,6 +261,7 @@ type WorkflowTaskArgs struct { VersionArgs *VersionArgs `bson:"version_args,omitempty" json:"version_args,omitempty"` ReqID string `bson:"req_id" json:"req_id"` RegistryID string `bson:"registry_id,omitempty" json:"registry_id,omitempty"` + StorageID string `bson:"storage_id,omitempty" json:"storage_id,omitempty"` DistributeEnabled bool `bson:"distribute_enabled" json:"distribute_enabled"` WorklowTaskCreator string `bson:"workflow_task_creator" json:"workflow_task_creator"` // Ignore docker build cache @@ -276,6 +278,7 @@ type WorkflowTaskArgs struct { Source string `bson:"source" json:"source"` CodehostID int `bson:"codehost_id" json:"codehost_id"` RepoOwner string `bson:"repo_owner" json:"repo_owner"` + RepoNamespace string `bson:"repo_namespace" json:"repo_namespace"` RepoName string `bson:"repo_name" json:"repo_name"` //github check run @@ -285,30 +288,24 @@ type WorkflowTaskArgs struct { IsParallel bool `json:"is_parallel" bson:"is_parallel"` } -// TargetArgs ... type TargetArgs struct { - // 服务组件名称 - Name string `bson:"name" json:"name"` - // 服务名称 - ServiceName string `bson:"service_name" json:"service_name"` - Build *BuildArgs `bson:"build" json:"build"` - Version string `bson:"version" json:"version"` - Deploy []DeployEnv `bson:"deloy" json:"deploy"` - Image string `bson:"image" json:"image"` - BinFile string `bson:"bin_file" json:"bin_file"` - Envs []*KeyVal `bson:"envs" json:"envs"` - HasBuild bool `bson:"has_build" json:"has_build"` - JenkinsBuildArgs *JenkinsBuildArgs `bson:"jenkins_build_args" json:"jenkins_build_args"` + Name string `bson:"name" json:"name"` + ImageName string `bson:"image_name" json:"image_name"` + ServiceName string `bson:"service_name" json:"service_name"` + ServiceType string `bson:"service_type,omitempty" json:"service_type,omitempty"` + ProductName string `bson:"product_name" json:"product_name"` + Build *BuildArgs `bson:"build" json:"build"` + Deploy []DeployEnv `bson:"deploy" json:"deploy"` + Image string `bson:"image,omitempty" json:"image,omitempty"` + BinFile string `bson:"bin_file" json:"bin_file"` + Envs []*KeyVal `bson:"envs" json:"envs"` + HasBuild bool `bson:"has_build" json:"has_build"` + JenkinsBuildArgs *JenkinsBuildArgs `bson:"jenkins_build_args,omitempty" json:"jenkins_build_args,omitempty"` } type JenkinsBuildArgs struct { - JobName string `bson:"job_name" json:"job_name"` - JenkinsBuildParams []*JenkinsBuildParam `bson:"jenkins_build_param" json:"jenkins_build_params"` -} - -type JenkinsBuildParam struct { - Name string `json:"name"` - Value interface{} `json:"value"` + JobName string `bson:"job_name" json:"job_name"` + JenkinsBuildParams []*types.JenkinsBuildParam `bson:"jenkins_build_param" json:"jenkins_build_params"` } type BuildArgs struct { @@ -317,6 +314,7 @@ type BuildArgs struct { type ArtifactArgs struct { Name string `bson:"name" json:"name"` + ImageName string `bson:"image_name,omitempty" json:"image_name,omitempty"` ServiceName string `bson:"service_name" json:"service_name"` Image string `bson:"image" json:"image"` Deploy []DeployEnv `bson:"deloy" json:"deploy"` @@ -422,25 +420,12 @@ type RenderKV struct { // KeyState can be new, deleted, normal type KeyState string -const ( - // KeyStateNew ... - KeyStateNew = KeyState("new") - // KeyStateUnused ... - KeyStateUnused = KeyState("unused") - // KeyStatePresent ... - KeyStatePresent = KeyState("present") -) - type Pipeline struct { - // ID is auto generated by mgo, ready only - ID primitive.ObjectID `bson:"_id" json:"id"` - Name string `bson:"name" json:"name"` - Type PipelineType `bson:"type" json:"type"` - Enabled bool `bson:"enabled" json:"enabled"` - TeamName string `bson:"team" json:"team"` - ProductName string `bson:"product_name" json:"product_name"` - // OrgID 单租户ID - OrgID int `bson:"org_id" json:"org_id"` + Name string `bson:"name" json:"name"` + Type PipelineType `bson:"type" json:"type"` + Enabled bool `bson:"enabled" json:"enabled"` + TeamName string `bson:"team" json:"team"` + ProductName string `bson:"product_name" json:"product_name"` // target 服务名称, k8s为容器名称, 物理机为服务名 Target string `bson:"target" json:"target"` // 使用预定义编译管理模块中的内容生成SubTasks, @@ -453,17 +438,11 @@ type Pipeline struct { CreateTime int64 `bson:"create_time" json:"create_time,omitempty"` UpdateTime int64 `bson:"update_time" json:"update_time,omitempty"` Schedules ScheduleCtrl `bson:"schedules,omitempty" json:"schedules,omitempty"` - Hook *Hook `bson:"hook,omitempty" json:"hook,omitempty"` Notifiers []string `bson:"notifiers,omitempty" json:"notifiers,omitempty"` RunCount int `bson:"run_count,omitempty" json:"run_count,omitempty"` DailyRunCount float32 `bson:"daily_run_count,omitempty" json:"daily_run_count,omitempty"` PassRate float32 `bson:"pass_rate,omitempty" json:"pass_rate,omitempty"` IsDeleted bool `bson:"is_deleted" json:"is_deleted"` - Slack *Slack `bson:"slack" json:"slack"` - NotifyCtl *NotifyCtl `bson:"notify_ctl" json:"notify_ctl"` - IsFavorite bool `bson:"-" json:"is_favorite"` - // 是否允许同时运行多次 - MultiRun bool `bson:"multi_run" json:"multi_run"` } // ScheduleCtrl ... @@ -488,188 +467,7 @@ const ( ServiceType = PipelineType("service") ) -type Hook struct { - Enabled bool `bson:"enabled" json:"enabled"` - GitHooks []GitHook `bson:"git_hooks" json:"git_hooks,omitempty"` -} - -// GitHook Events: push, pull_request -// MatchFolders: 包含目录或者文件后缀 -// 以!开头的目录或者后缀名为不运行pipeline的过滤条件 -type GitHook struct { - Owner string `bson:"repo_owner" json:"repo_owner"` - Repo string `bson:"repo" json:"repo"` - Branch string `bson:"branch" json:"branch"` - Events []string `bson:"events" json:"events"` - MatchFolders []string `bson:"match_folders" json:"match_folders,omitempty"` - CodehostID int `bson:"codehost_id" json:"codehost_id"` - AutoCancel bool `bson:"auto_cancel" json:"auto_cancel"` -} - -type Slack struct { - Channel string `bson:"channel" json:"channel"` - Enabled bool `bson:"enabled" json:"enabled"` - Notifiers []string `bson:"notifiers" json:"notifiers"` - NotifyType SlackNotifyType `bson:"notify_type" json:"notify_type"` -} - -// SlackNotifyType ... -type SlackNotifyType string - -const ( - // SlackAll ... - SlackAll = SlackNotifyType("all") - // SlackOnChange ... - SlackOnChange = SlackNotifyType("onchange") - // SlackOnfailure ... - SlackOnfailure = SlackNotifyType("onfailure") -) - -type NotifyCtl struct { - Enabled bool `bson:"enabled" json:"enabled"` - WebHookType string `bson:"webhook_type" json:"webhook_type"` - WeChatWebHook string `bson:"weChat_webHook,omitempty" json:"weChat_webHook,omitempty"` - DingDingWebHook string `bson:"dingding_webhook,omitempty" json:"dingding_webhook,omitempty"` - FeiShuWebHook string `bson:"feishu_webhook,omitempty" json:"feishu_webhook,omitempty"` - AtMobiles []string `bson:"at_mobiles,omitempty" json:"at_mobiles,omitempty"` - IsAtAll bool `bson:"is_at_all,omitempty" json:"is_at_all,omitempty"` - NotifyTypes []string `bson:"notify_type" json:"notify_type"` -} - type Workflow struct { - ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` - Name string `bson:"name" json:"name"` - Enabled bool `bson:"enabled" json:"enabled"` - ProductTmplName string `bson:"product_tmpl_name" json:"product_tmpl_name"` - Team string `bson:"team,omitempty" json:"team,omitempty"` - EnvName string `bson:"env_name" json:"env_name,omitempty"` - Description string `bson:"description,omitempty" json:"description,omitempty"` - UpdateBy string `bson:"update_by" json:"update_by,omitempty"` - CreateBy string `bson:"create_by" json:"create_by,omitempty"` - UpdateTime int64 `bson:"update_time" json:"update_time"` - CreateTime int64 `bson:"create_time" json:"create_time"` - Schedules *ScheduleCtrl `bson:"schedules,omitempty" json:"schedules,omitempty"` - ScheduleEnabled bool `bson:"schedule_enabled" json:"schedule_enabled"` - Slack *Slack `bson:"slack,omitempty" json:"slack,omitempty"` - BuildStage *BuildStage `bson:"build_stage" json:"build_stage"` - ArtifactStage *ArtifactStage `bson:"artifact_stage,omitempty" json:"artifact_stage,omitempty"` - TestStage *TestStage `bson:"test_stage" json:"test_stage"` - SecurityStage *SecurityStage `bson:"security_stage" json:"security_stage"` - DistributeStage *DistributeStage `bson:"distribute_stage" json:"distribute_stage"` - NotifyCtl *NotifyCtl `bson:"notify_ctl,omitempty" json:"notify_ctl,omitempty"` - HookCtl *HookCtrl `bson:"hook_ctl" json:"hook_ctl"` - IsFavorite bool `bson:"-" json:"is_favorite"` - LastestTask *TaskInfo `bson:"-" json:"lastest_task"` - LastSucessTask *TaskInfo `bson:"-" json:"last_task_success"` - LastFailureTask *TaskInfo `bson:"-" json:"last_task_failure"` - TotalDuration int64 `bson:"-" json:"total_duration"` - TotalNum int `bson:"-" json:"total_num"` - TotalSuccess int `bson:"-" json:"total_success"` - - // ResetImage indicate whether reset image to original version after completion - ResetImage bool `json:"reset_image" bson:"reset_image"` - // IsParallel 控制单一工作流的任务是否支持并行处理 - IsParallel bool `json:"is_parallel" bson:"is_parallel"` -} - -// TaskInfo ... -type TaskInfo struct { - TaskID int64 `bson:"task_id" json:"task_id"` - PipelineName string `bson:"pipeline_name" json:"pipeline_name"` - Status Status `bson:"status" json:"status"` -} - -type Status string - -type BuildStage struct { - Enabled bool `bson:"enabled" json:"enabled"` - Modules []*BuildModule `bson:"modules" json:"modules"` + Name string `json:"name"` + Schedules *ScheduleCtrl `json:"schedules,omitempty"` } - -type BuildModule struct { - Target *ServiceModuleTarget `bson:"target" json:"target"` - BuildModuleVer string `bson:"build_module_ver" json:"build_module_ver"` -} - -type ServiceModuleTarget struct { - ProductName string `bson:"product_name" json:"product_name"` - ServiceName string `bson:"service_name" json:"service_name"` - ServiceModule string `bson:"service_module" json:"service_module"` -} - -type ArtifactStage struct { - Enabled bool `bson:"enabled" json:"enabled"` - Modules []*ArtifactModule `bson:"modules" json:"modules"` -} - -type ArtifactModule struct { - Target *ServiceModuleTarget `bson:"target" json:"target"` -} - -type TestStage struct { - Enabled bool `bson:"enabled" json:"enabled"` - TestNames []string `bson:"test_names,omitempty" json:"test_names,omitempty"` - Tests []*TestExecArgs `bson:"tests,omitempty" json:"tests,omitempty"` -} - -type TestExecArgs struct { - Name string `bson:"test_name" json:"test_name"` - Envs []*KeyVal `bson:"envs" json:"envs"` -} - -type SecurityStage struct { - Enabled bool `bson:"enabled" json:"enabled"` -} - -type DistributeStage struct { - Enabled bool `bson:"enabled" json:"enabled"` - S3StorageID string `bson:"s3_storage_id" json:"s3_storage_id"` - ImageRepo string `bson:"image_repo" json:"image_repo"` - JumpBoxHost string `bson:"jump_box_host" json:"jump_box_host"` - Distributes []*ProductDistribute `bson:"distributes" json:"distributes"` - - // repos to release images - Releases []RepoImage `bson:"releases" json:"releases"` -} - -type RepoImage struct { - RepoID string `json:"repo_id" bson:"repo_id"` - Name string `json:"name" bson:"name" yaml:"name"` - Username string `json:"-" yaml:"username"` - Password string `json:"-" yaml:"password"` - Host string `json:"host" yaml:"host"` - Namespace string `json:"namespace" yaml:"namespace"` -} - -type ProductDistribute struct { - Target *ServiceModuleTarget `bson:"target" json:"target"` - ImageDistribute bool `bson:"image_distribute" json:"image_distribute"` - JumpBoxDistribute bool `bson:"jump_box_distribute" json:"jump_box_distribute"` - QstackDistribute bool `bson:"qstack_distribute" json:"qstack_distribute"` -} - -type HookCtrl struct { - Enabled bool `bson:"enabled" json:"enabled"` - Items []*WorkflowHook `bson:"items" json:"items"` -} - -type WorkflowHook struct { - AutoCancel bool `bson:"auto_cancel" json:"auto_cancel"` - CheckPatchSetChange bool `bson:"check_patch_set_change" json:"check_patch_set_change"` - MainRepo MainHookRepo `bson:"main_repo" json:"main_repo"` - WorkflowArgs *WorkflowTaskArgs `bson:"workflow_args" json:"workflow_args"` -} - -type MainHookRepo struct { - Source string `bson:"source,omitempty" json:"source,omitempty"` - RepoOwner string `bson:"repo_owner" json:"repo_owner"` - RepoName string `bson:"repo_name" json:"repo_name"` - Branch string `bson:"branch" json:"branch"` - MatchFolders []string `bson:"match_folders" json:"match_folders,omitempty"` - CodehostID int `bson:"codehost_id" json:"codehost_id"` - Events []HookEventType `bson:"events" json:"events"` - Label string `bson:"label" json:"label"` - Revision string `bson:"revision" json:"revision"` -} - -type HookEventType string diff --git a/pkg/microservice/cron/server/server.go b/pkg/microservice/cron/server/server.go index e35fc751145a976068c3f87006eac988745d9aa2..85e635053a466d797930911e0d4a1c1b3897eb88 100644 --- a/pkg/microservice/cron/server/server.go +++ b/pkg/microservice/cron/server/server.go @@ -39,6 +39,9 @@ func Serve(ctx context.Context) error { cronClient := scheduler.NewCronClient() cronClient.Init() + cronV3Client := scheduler.NewCronV3() + cronV3Client.Start() + http.HandleFunc("/ping", ping) server := &http.Server{Addr: ":8091", Handler: nil} diff --git a/pkg/microservice/hubagent/config/config.go b/pkg/microservice/hubagent/config/config.go index f46e9ccc7a7390614939c12bcf970ecd447a8ed9..76968ace53e9e9962db055e6e6bdd9d576d0b887 100644 --- a/pkg/microservice/hubagent/config/config.go +++ b/pkg/microservice/hubagent/config/config.go @@ -39,3 +39,7 @@ func KubernetesServiceHost() string { func KubernetesServicePort() string { return viper.GetString(setting.KubernetesServicePort) } + +func AslanBaseAddr() string { + return viper.GetString(setting.AslanBaseAddr) +} diff --git a/pkg/microservice/hubagent/core/service/client.go b/pkg/microservice/hubagent/core/service/client.go index e1b53b71a2302a74bca4efb0c9281da0a1e348e7..5a7edcd688c4eadcb504db3fc6590f7a6af6dcb7 100644 --- a/pkg/microservice/hubagent/core/service/client.go +++ b/pkg/microservice/hubagent/core/service/client.go @@ -17,6 +17,7 @@ limitations under the License. package service import ( + "context" "encoding/base64" "encoding/json" "fmt" @@ -25,12 +26,11 @@ import ( "strings" "time" - "github.com/cenkalti/backoff/v3" + "github.com/cenkalti/backoff/v4" "github.com/gorilla/websocket" "github.com/pkg/errors" "go.uber.org/zap" - commonconfig "github.com/koderover/zadig/pkg/config" "github.com/koderover/zadig/pkg/microservice/hubagent/config" "github.com/koderover/zadig/pkg/setting" "github.com/koderover/zadig/pkg/tool/log" @@ -97,13 +97,6 @@ func (c *Client) getParams() (*input, error) { } func Init() error { - log.Init(&log.Config{ - Level: commonconfig.LogLevel(), - Filename: commonconfig.LogFile(), - SendToFile: commonconfig.SendLogToFile(), - Development: commonconfig.Mode() != setting.ReleaseMode, - }) - token := config.HubAgentToken() if token == "" { return fmt.Errorf("token must be configured") @@ -173,6 +166,7 @@ func (c *Client) Start() error { tm := time.Now() remotedialer.ClientConnect( + context.Background(), connectURL, headers, &websocket.Dialer{ diff --git a/pkg/microservice/hubagent/server/rest/router.go b/pkg/microservice/hubagent/server/rest/router.go new file mode 100644 index 0000000000000000000000000000000000000000..701339630cd41c91470d48c43f659235dc05aa0f --- /dev/null +++ b/pkg/microservice/hubagent/server/rest/router.go @@ -0,0 +1,42 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rest + +import ( + "github.com/gin-gonic/gin" + + cachehandler "github.com/koderover/zadig/pkg/handler/cache" + "github.com/koderover/zadig/pkg/microservice/user/core/handler" +) + +func (s *engine) injectRouterGroup(router *gin.RouterGroup) { + for _, r := range []injector{ + new(handler.Router), + } { + r.Inject(router.Group("/api/v1")) + } + + for name, r := range map[string]injector{ + "/api/cache": cachehandler.NewRouter(), + } { + r.Inject(router.Group(name)) + } +} + +type injector interface { + Inject(router *gin.RouterGroup) +} diff --git a/pkg/microservice/hubagent/server/rest/server.go b/pkg/microservice/hubagent/server/rest/server.go new file mode 100644 index 0000000000000000000000000000000000000000..c2ef16b24873bc3e409588b6f95e146eea1294cd --- /dev/null +++ b/pkg/microservice/hubagent/server/rest/server.go @@ -0,0 +1,76 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rest + +import ( + "net/http" + + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/config" + ginmiddleware "github.com/koderover/zadig/pkg/middleware/gin" + "github.com/koderover/zadig/pkg/tool/log" +) + +type engine struct { + *gin.Engine + + mode string +} + +func NewEngine() *engine { + s := &engine{mode: config.Mode()} + + gin.SetMode(s.mode) + + s.injectMiddlewares() + s.injectRouters() + + return s +} + +func (s *engine) injectMiddlewares() { + g := gin.New() + defer func() { + s.Engine = g + }() + + if s.mode == gin.TestMode { + return + } + g.Use(ginmiddleware.Response()) + g.Use(ginmiddleware.RequestID()) + g.Use(ginmiddleware.RequestLog(log.NewFileLogger(config.RequestLogFile()))) + g.Use(gin.Recovery()) +} + +func (s *engine) injectRouters() { + g := s.Engine + + g.NoRoute(func(c *gin.Context) { + c.String(http.StatusNotFound, "Invalid path: %s", c.Request.URL.Path) + }) + g.HandleMethodNotAllowed = true + g.NoMethod(func(c *gin.Context) { + c.String(http.StatusMethodNotAllowed, "Method not allowed: %s %s", c.Request.Method, c.Request.URL.Path) + }) + + apiRouters := g.Group("") + s.injectRouterGroup(apiRouters) + + s.Engine = g +} diff --git a/pkg/microservice/hubagent/server/server.go b/pkg/microservice/hubagent/server/server.go index e76babad5d209f3920e71c2581c1668abc7ffe06..415bf01c2d8ad48ea3d1cdfb8cf1744d0336d6f3 100644 --- a/pkg/microservice/hubagent/server/server.go +++ b/pkg/microservice/hubagent/server/server.go @@ -17,13 +17,94 @@ limitations under the License. package server import ( + "context" + "net/http" + "time" + + "github.com/koderover/zadig/pkg/config" + config2 "github.com/koderover/zadig/pkg/microservice/hubagent/config" "github.com/koderover/zadig/pkg/microservice/hubagent/core/service" + "github.com/koderover/zadig/pkg/microservice/hubagent/server/rest" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/client/aslan" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" + "github.com/koderover/zadig/pkg/tool/log" + registrytool "github.com/koderover/zadig/pkg/tool/registries" ) -func Serve() error { +func init() { + log.Init(&log.Config{ + Level: config.LogLevel(), + Filename: config.LogFile(), + SendToFile: config.SendLogToFile(), + Development: config.Mode() != setting.ReleaseMode, + }) +} + +func Serve(ctx context.Context) error { + log.Info("Start Hub-Agent service.") + + engine := rest.NewEngine() + server := &http.Server{Addr: ":80", Handler: engine} + + initDinD() + + go func() { + <-ctx.Done() + + ctx, cancel := context.WithTimeout(context.TODO(), 5*time.Second) + defer cancel() + + if err := server.Shutdown(ctx); err != nil { + log.Errorf("Failed to stop server, error: %s", err) + } + }() + + go func() { + err := server.ListenAndServe() + if err != nil && err != http.ErrServerClosed { + log.Errorf("Failed to start http server, error: %s", err) + return + } + }() + if err := service.Init(); err != nil { return err } return nil } + +func initDinD() { + client := aslan.NewExternal(config2.AslanBaseAddr(), "") + + ls, err := client.ListRegistries() + if err != nil { + log.Fatalf("failed to get information from zadig server to set DinD, err: %s", err) + } + + regList := make([]*registrytool.RegistryInfoForDinDUpdate, 0) + for _, reg := range ls { + regItem := ®istrytool.RegistryInfoForDinDUpdate{ + ID: reg.ID, + RegAddr: reg.RegAddr, + } + if reg.AdvancedSetting != nil { + regItem.AdvancedSetting = ®istrytool.RegistryAdvancedSetting{ + TLSEnabled: reg.AdvancedSetting.TLSEnabled, + TLSCert: reg.AdvancedSetting.TLSCert, + } + } + regList = append(regList, regItem) + } + + dynamicClient, err := kubeclient.GetDynamicKubeClient(config2.AslanBaseAddr(), setting.LocalClusterID) + if err != nil { + log.Fatalf("failed to create dynamic kubernetes clientset for clusterID: %s, the error is: %s", setting.LocalClusterID, err) + } + + err = registrytool.PrepareDinD(dynamicClient, "koderover-agent", regList) + if err != nil { + log.Fatalf("failed to update dind, the error is: %s", err) + } +} diff --git a/pkg/microservice/hubserver/core/repository/models/models.go b/pkg/microservice/hubserver/core/repository/models/models.go index 4bfefe013db24ce09ce1143aeb05ca85d37390d2..50edf38f39a66c5561d8eee384d172b883950730 100644 --- a/pkg/microservice/hubserver/core/repository/models/models.go +++ b/pkg/microservice/hubserver/core/repository/models/models.go @@ -30,20 +30,22 @@ type K8SClusterInfo struct { } type K8SCluster struct { - ID primitive.ObjectID `json:"id" bson:"_id"` - Name string `json:"name" bson:"name"` - Tags []string `json:"tags" bson:"tags"` - Description string `json:"description" bson:"description"` - Namespace string `json:"namespace" bson:"namespace"` - Info *K8SClusterInfo `json:"info,omitempty" bson:"info,omitempty"` - Status config.K8SClusterStatus `json:"status" bson:"status"` - Error string `json:"error" bson:"error"` - Yaml string `json:"yaml" bson:"yaml"` - Production bool `json:"production" bson:"production"` - CreatedAt int64 `json:"createdAt" bson:"createdAt"` - CreatedBy string `json:"createdBy" bson:"createdBy"` - Disconnected bool `json:"-" bson:"disconnected"` - Token string `json:"token" bson:"-"` + ID primitive.ObjectID `json:"id" bson:"_id"` + Name string `json:"name" bson:"name"` + Tags []string `json:"tags" bson:"tags"` + Description string `json:"description" bson:"description"` + Namespace string `json:"namespace" bson:"namespace"` + Info *K8SClusterInfo `json:"info,omitempty" bson:"info,omitempty"` + Status config.K8SClusterStatus `json:"status" bson:"status"` + Error string `json:"error" bson:"error"` + Yaml string `json:"yaml" bson:"yaml"` + Production bool `json:"production" bson:"production"` + CreatedAt int64 `json:"createdAt" bson:"createdAt"` + CreatedBy string `json:"createdBy" bson:"createdBy"` + Disconnected bool `json:"-" bson:"disconnected"` + Token string `json:"token" bson:"-"` + Local bool `json:"local" bson:"local"` + LastConnectionTime int64 `json:"last_connection_time" bson:"last_connection_time"` } func (K8SCluster) TableName() string { diff --git a/pkg/microservice/hubserver/core/repository/mongodb/k8scluster.go b/pkg/microservice/hubserver/core/repository/mongodb/k8scluster.go index 9993a2d45f4b73d74ce96ebc845c4606edecda06..eb1539894fb697c348cfc25140fdd8e06d6a8aaf 100644 --- a/pkg/microservice/hubserver/core/repository/mongodb/k8scluster.go +++ b/pkg/microservice/hubserver/core/repository/mongodb/k8scluster.go @@ -18,6 +18,7 @@ package mongodb import ( "context" + "time" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" @@ -68,7 +69,8 @@ func (c *K8sClusterColl) UpdateStatus(cluster *models.K8SCluster) error { query := bson.M{"_id": cluster.ID} update := bson.M{"$set": bson.M{ - "status": cluster.Status, + "last_connection_time": cluster.LastConnectionTime, + "status": cluster.Status, }} _, err := c.UpdateOne(context.TODO(), query, update) @@ -88,6 +90,7 @@ func (c *K8sClusterColl) UpdateConnectState(id string, disconnected bool) error if disconnected { change["status"] = config.Disconnected } else { + change["last_connection_time"] = time.Now().Unix() change["status"] = config.Pending } update := bson.M{"$set": change} diff --git a/pkg/microservice/hubserver/core/service/service.go b/pkg/microservice/hubserver/core/service/service.go index 87aba25f070f3c4d6551bd43dfdcf3aa6b2d6160..8b7c8ab6ed771926146244dfd68009cd4b6cf64d 100644 --- a/pkg/microservice/hubserver/core/service/service.go +++ b/pkg/microservice/hubserver/core/service/service.go @@ -17,8 +17,6 @@ limitations under the License. package service import ( - "crypto/tls" - "crypto/x509" "encoding/base64" "encoding/json" "fmt" @@ -29,6 +27,7 @@ import ( "github.com/gorilla/mux" "k8s.io/apimachinery/pkg/util/proxy" + "k8s.io/apimachinery/pkg/util/wait" "github.com/koderover/zadig/pkg/microservice/hubserver/config" "github.com/koderover/zadig/pkg/microservice/hubserver/core/repository/models" @@ -84,13 +83,12 @@ func Authorize(req *http.Request) (clientKey string, authed bool, err error) { clusters.Store(cluster.ID.Hex(), input.Cluster) - if cluster.Status != "normal" { - cluster.Status = "normal" - err = mongodb.NewK8sClusterColl().UpdateStatus(cluster) - if err != nil { - log.Errorf("failed to update clusters status %s %v", cluster.Name, err) - return - } + cluster.Status = "normal" + cluster.LastConnectionTime = time.Now().Unix() + err = mongodb.NewK8sClusterColl().UpdateStatus(cluster) + if err != nil { + log.Errorf("failed to update clusters status %s %v", cluster.Name, err) + return } log.Infof("cluster %s connected", cluster.Name) @@ -171,12 +169,24 @@ func Forward(server *remotedialer.Server, w http.ResponseWriter, r *http.Request } }() - _, err = mongodb.NewK8sClusterColl().Get(clientKey) - if err != nil { - return - } + //_, err = mongodb.NewK8sClusterColl().Get(clientKey) + //if err != nil { + // return + //} clusterInfo, exists := clusters.Load(clientKey) + if !server.HasSession(clientKey) || !exists { + for i := 0; i < 4; i++ { + log.Infof("stuck waiting for connection index:%d", i) + if server.HasSession(clientKey) && exists { + log.Infof("succeeded waiting for connection index:%d", i) + break + } + time.Sleep(wait.Jitter(3*time.Second, 2)) + clusterInfo, exists = clusters.Load(clientKey) + } + } + if !server.HasSession(clientKey) || !exists { errHandled = true logger.Infof("waiting for cluster %s to connect", clientKey) @@ -204,23 +214,12 @@ func Forward(server *remotedialer.Server, w http.ResponseWriter, r *http.Request r.URL.Host = r.Host r.Header.Set("authorization", "Bearer "+cluster.Token) - var certBytes []byte - certBytes, err = base64.StdEncoding.DecodeString(cluster.CACert) + transport, err := server.GetTransport(cluster.CACert, clientKey) if err != nil { + log.Errorf(fmt.Sprintf("failed to get transport, err %s", err)) return } - certs := x509.NewCertPool() - certs.AppendCertsFromPEM(certBytes) - - transport := &http.Transport{} - - //noinspection GoDeprecation - transport.Dial = server.Dialer(clientKey, 15*time.Second) - transport.TLSClientConfig = &tls.Config{ - RootCAs: certs, - } - httpProxy := proxy.NewUpgradeAwareHandler(endpoint, transport, true, false, er) httpProxy.ServeHTTP(w, r) } @@ -235,7 +234,7 @@ func Reset() { } for _, cluster := range clusters { - if cluster.Status == config.Normal { + if cluster.Status == config.Normal && !cluster.Local { cluster.Status = config.Abnormal err := mongodb.NewK8sClusterColl().UpdateStatus(cluster) if err != nil { @@ -267,11 +266,12 @@ func Sync(server *remotedialer.Server, stopCh <-chan struct{}) { "cluster %s connected changed %s => %s", cluster.Name, cluster.Status, config.Normal, ) + cluster.LastConnectionTime = time.Now().Unix() cluster.Status = config.Normal statusChanged = true } } else { - if cluster.Status == config.Normal { + if cluster.Status == config.Normal && !cluster.Local { log.Infof( "cluster %s disconnected changed %s => %s", cluster.Name, cluster.Status, config.Abnormal, diff --git a/pkg/microservice/jenkinsplugin/core/service/jenkins.go b/pkg/microservice/jenkinsplugin/core/service/jenkins.go index bbd4bc662bf88c89d68ab2ed6b3e03dfa03e4e12..a633cf75dbab3d40b8f2553e087aab9a7017ef92 100644 --- a/pkg/microservice/jenkinsplugin/core/service/jenkins.go +++ b/pkg/microservice/jenkinsplugin/core/service/jenkins.go @@ -27,7 +27,7 @@ import ( "time" "github.com/bndr/gojenkins" - yaml "gopkg.in/yaml.v3" + "gopkg.in/yaml.v3" "github.com/koderover/zadig/pkg/microservice/jenkinsplugin/config" "github.com/koderover/zadig/pkg/tool/log" @@ -194,5 +194,5 @@ func getBuild(ctx context.Context, buildID int64, jenkins *gojenkins.Jenkins, jo if status == 200 { return &build, nil } - return nil, nil + return nil, fmt.Errorf("fail to get build") } diff --git a/pkg/microservice/jobexecutor/config/config.go b/pkg/microservice/jobexecutor/config/config.go new file mode 100644 index 0000000000000000000000000000000000000000..d75f73a51f5788c323c176c7bf0b327040f13f60 --- /dev/null +++ b/pkg/microservice/jobexecutor/config/config.go @@ -0,0 +1,40 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "github.com/koderover/zadig/pkg/setting" + "github.com/spf13/viper" + + _ "github.com/koderover/zadig/pkg/config" +) + +func JobConfigFile() string { + return viper.GetString(setting.JobConfigFile) +} + +func Path() string { + return viper.GetString(setting.Path) +} + +func DockerHost() string { + return viper.GetString(setting.DockerHost) +} + +func Home() string { + return viper.GetString(setting.Home) +} diff --git a/pkg/microservice/jobexecutor/core/service/cmd/cmd.go b/pkg/microservice/jobexecutor/core/service/cmd/cmd.go new file mode 100644 index 0000000000000000000000000000000000000000..cc983badd8065a534a3c1e60b1071701d917b03c --- /dev/null +++ b/pkg/microservice/jobexecutor/core/service/cmd/cmd.go @@ -0,0 +1,28 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import "os/exec" + +// Command ... +type Command struct { + Cmd *exec.Cmd + // DisableTrace display command args + DisableTrace bool + // IgnoreError ingore command run error + IgnoreError bool +} diff --git a/pkg/microservice/jobexecutor/core/service/cmd/git.go b/pkg/microservice/jobexecutor/core/service/cmd/git.go new file mode 100644 index 0000000000000000000000000000000000000000..243aa1dc001df96503ecc6f445da09e8ca9eda4c --- /dev/null +++ b/pkg/microservice/jobexecutor/core/service/cmd/git.go @@ -0,0 +1,154 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import "os/exec" + +// InitGit creates an empty git repository. +// it returns command git init +func InitGit(dir string) *exec.Cmd { + cmd := exec.Command( + "git", + "init", + ) + cmd.Dir = dir + return cmd +} + +// RemoteAdd sets the remote origin for the repository. +func RemoteAdd(remoteName, remote string) *exec.Cmd { + return exec.Command( + "git", + "remote", + "add", + remoteName, + remote, + ) +} + +// RemoteRemove removes the remote origin for the repository. +func RemoteRemove(remoteName string) *exec.Cmd { + return exec.Command( + "git", + "remote", + "remove", + remoteName, + ) +} + +// CheckoutHead returns command git checkout -qf FETCH_HEAD +func CheckoutHead() *exec.Cmd { + return exec.Command( + "git", + "checkout", + "-qf", + "FETCH_HEAD", + ) +} + +// Fetch fetches changes by ref, ref can be a tag, branch or pr. --depth=1 is used to limit fetching +// to the last commit from the tip of each remote branch history. +// e.g. git fetch origin +refs/heads/onboarding --depth=1 +func Fetch(remoteName, ref string) *exec.Cmd { + return exec.Command( + "git", + "fetch", + remoteName, + "+"+ref, // "+" means overwrite + "--depth=1", + ) +} + +// DeepenedFetch deepens the fetch history. It is similar with Fetch but accepts 500 more commit history than +// last Fetch operation by --deepen=500 option. +// e.g. git fetch origin +refs/heads/onboarding --deepen=500 +func DeepenedFetch(remoteName, ref string) *exec.Cmd { + return exec.Command( + "git", + "fetch", + remoteName, + "+"+ref, // "+" means overwrite + "--deepen=500", + ) +} + +// ResetMerge reset last merge +// It return command git reset --merge +func ResetMerge() *exec.Cmd { + return exec.Command( + "git", + "reset", + "--merge", + ) + +} + +// Merge merge a branch +// e.g. git merge demo +func Merge(branch string) *exec.Cmd { + return exec.Command( + "git", + "merge", + branch, + "--allow-unrelated-histories", // add this option to avoid "fatal: refusing to merge unrelated histories" + ) +} + +// UpdateSubmodules returns command: git submodule update --init --recursive +func UpdateSubmodules() *exec.Cmd { + cmd := exec.Command( + "git", + "submodule", + "update", + "--init", + "--recursive", + ) + return cmd +} + +// SetConfig returns command: git config --global $KEY $VA +// e.g. git config --global user.name username +func SetConfig(key, value string) *exec.Cmd { + return exec.Command( + "git", + "config", + "--global", + key, + value, + ) +} + +// SetConfig returns command: git config --global $KEY $VA +// e.g. git config --global user.name username +func Gc() *exec.Cmd { + return exec.Command( + "git", + "gc", + ) +} + +// ShowLastLog returns command git --no-pager log --oneline -1 +// It shows last commit messge with sha +func ShowLastLog() *exec.Cmd { + return exec.Command( + "git", + "--no-pager", + "log", + "--oneline", + "-1", + ) +} diff --git a/pkg/microservice/jobexecutor/core/service/job.go b/pkg/microservice/jobexecutor/core/service/job.go new file mode 100644 index 0000000000000000000000000000000000000000..457935a5c36fc2c6ffe4302d2386eaca9e979ec8 --- /dev/null +++ b/pkg/microservice/jobexecutor/core/service/job.go @@ -0,0 +1,168 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package job + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "time" + + "github.com/koderover/zadig/pkg/microservice/jobexecutor/config" + "github.com/koderover/zadig/pkg/microservice/jobexecutor/core/service/meta" + "github.com/koderover/zadig/pkg/microservice/jobexecutor/core/service/step" + "github.com/koderover/zadig/pkg/types/job" + "gopkg.in/yaml.v3" +) + +type Job struct { + Ctx *meta.JobContext + StartTime time.Time + ActiveWorkspace string + UserEnvs map[string]string +} + +const ( + // MaxContainerTerminationMessageLength is the upper bound any one container may write to + // its termination message path. Contents above this length will cause a failure. + MaxContainerTerminationMessageLength = 1024 * 4 +) + +func NewJob() (*Job, error) { + context, err := ioutil.ReadFile(config.JobConfigFile()) + if err != nil { + return nil, fmt.Errorf("read job config file error: %v", err) + } + + var ctx *meta.JobContext + if err := yaml.Unmarshal(context, &ctx); err != nil { + return nil, fmt.Errorf("cannot unmarshal job data: %v", err) + } + + if ctx.Paths != "" { + ctx.Paths = fmt.Sprintf("%s:%s", config.Path(), ctx.Paths) + } else { + ctx.Paths = config.Path() + } + + job := &Job{ + Ctx: ctx, + } + + err = job.EnsureActiveWorkspace(ctx.Workspace) + if err != nil { + return nil, fmt.Errorf("failed to ensure active workspace `%s`: %s", ctx.Workspace, err) + } + + userEnvs := job.getUserEnvs() + job.UserEnvs = make(map[string]string, len(userEnvs)) + for _, env := range userEnvs { + items := strings.Split(env, "=") + if len(items) != 2 { + continue + } + + job.UserEnvs[items[0]] = items[1] + } + + return job, nil +} + +func (j *Job) EnsureActiveWorkspace(workspace string) error { + if workspace == "" { + tempWorkspace, err := ioutil.TempDir(os.TempDir(), "jobexecutor") + if err != nil { + return fmt.Errorf("create workspace error: %v", err) + } + j.ActiveWorkspace = tempWorkspace + return os.Chdir(j.ActiveWorkspace) + } + + err := os.MkdirAll(workspace, os.ModePerm) + if err != nil { + return fmt.Errorf("failed to create workspace: %v", err) + } + j.ActiveWorkspace = workspace + + return os.Chdir(j.ActiveWorkspace) +} + +func (j *Job) getUserEnvs() []string { + envs := []string{ + "CI=true", + "ZADIG=true", + fmt.Sprintf("HOME=%s", config.Home()), + fmt.Sprintf("WORKSPACE=%s", j.ActiveWorkspace), + } + + j.Ctx.Paths = strings.Replace(j.Ctx.Paths, "$HOME", config.Home(), -1) + envs = append(envs, fmt.Sprintf("PATH=%s", j.Ctx.Paths)) + envs = append(envs, fmt.Sprintf("DOCKER_HOST=%s", config.DockerHost())) + envs = append(envs, j.Ctx.Envs...) + envs = append(envs, j.Ctx.SecretEnvs...) + + return envs +} + +func (j *Job) Run(ctx context.Context) error { + if err := os.MkdirAll(job.JobOutputDir, os.ModePerm); err != nil { + return err + } + if err := step.RunSteps(ctx, j.Ctx.Steps, j.ActiveWorkspace, j.Ctx.Paths, j.getUserEnvs(), j.Ctx.SecretEnvs); err != nil { + return err + } + return nil +} +func (j *Job) AfterRun(ctx context.Context) error { + return j.collectJobResult(ctx) +} + +func (j *Job) collectJobResult(ctx context.Context) error { + outputs := []*job.JobOutput{} + for _, outputName := range j.Ctx.Outputs { + fileContents, err := ioutil.ReadFile(filepath.Join(job.JobOutputDir, outputName)) + if os.IsNotExist(err) { + continue + } else if err != nil { + return err + } + outputs = append(outputs, &job.JobOutput{Name: outputName, Value: string(fileContents)}) + } + jsonOutput, err := json.Marshal(outputs) + if err != nil { + return err + } + + if len(jsonOutput) > MaxContainerTerminationMessageLength { + return fmt.Errorf("termination message is above max allowed size 4096, caused by large task result") + } + + f, err := os.OpenFile(job.JobTerminationFile, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + return err + } + defer f.Close() + + if _, err = f.Write(jsonOutput); err != nil { + return err + } + return f.Sync() +} diff --git a/pkg/microservice/jobexecutor/core/service/meta/types.go b/pkg/microservice/jobexecutor/core/service/meta/types.go new file mode 100644 index 0000000000000000000000000000000000000000..f8644bb61ded20538b1901d91720a0273aa67d18 --- /dev/null +++ b/pkg/microservice/jobexecutor/core/service/meta/types.go @@ -0,0 +1,44 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package meta + +type JobContext struct { + Name string `yaml:"name"` + // Workspace 容器工作目录 [必填] + Workspace string `yaml:"workspace"` + // Envs 用户注入环境变量, 包括安装脚本环境变量 [optional] + Envs EnvVar `yaml:"envs"` + // SecretEnvs 用户注入敏感信息环境变量, value不能在stdout stderr中输出 [optional] + SecretEnvs EnvVar `yaml:"secret_envs"` + // WorkflowName + WorkflowName string `yaml:"workflow_name"` + // TaskID + TaskID int64 `yaml:"task_id"` + // Paths 执行脚本Path + Paths string `yaml:"paths"` + + Steps []*Step `yaml:"steps"` + Outputs []string `yaml:"outputs"` +} + +type Step struct { + Name string `yaml:"name"` + StepType string `yaml:"type"` + Spec interface{} `yaml:"spec"` +} + +type EnvVar []string diff --git a/pkg/microservice/jobexecutor/core/service/step/step.go b/pkg/microservice/jobexecutor/core/service/step/step.go new file mode 100644 index 0000000000000000000000000000000000000000..718f1fb685889d84108e4ab643087ceb156d02e6 --- /dev/null +++ b/pkg/microservice/jobexecutor/core/service/step/step.go @@ -0,0 +1,178 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package step + +import ( + "bufio" + "context" + "fmt" + "io" + "os" + "strings" + + "github.com/koderover/zadig/pkg/microservice/jobexecutor/config" + "github.com/koderover/zadig/pkg/microservice/jobexecutor/core/service/cmd" + "github.com/koderover/zadig/pkg/microservice/jobexecutor/core/service/meta" + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/util" +) + +type Step interface { + Run(ctx context.Context) error +} + +func RunSteps(ctx context.Context, steps []*meta.Step, workspace, paths string, envs, secretEnvs []string) error { + for _, stepInfo := range steps { + if err := runStep(ctx, stepInfo, workspace, paths, envs, secretEnvs); err != nil { + return err + } + } + return nil +} +func runStep(ctx context.Context, step *meta.Step, workspace, paths string, envs, secretEnvs []string) error { + var stepInstance Step + var err error + + switch step.StepType { + case "shell": + stepInstance, err = NewShellStep(step.Spec, workspace, paths, envs, secretEnvs) + if err != nil { + return err + } + case "git": + stepInstance, err = NewGitStep(step.Spec, workspace, envs, secretEnvs) + if err != nil { + return err + } + case "docker_build": + stepInstance, err = NewDockerBuildStep(step.Spec, workspace, envs, secretEnvs) + if err != nil { + return err + } + case "tools": + stepInstance, err = NewToolInstallStep(step.Spec, workspace, envs, secretEnvs) + if err != nil { + return err + } + case "archive": + stepInstance, err = NewArchiveStep(step.Spec, workspace, envs, secretEnvs) + if err != nil { + return err + } + default: + err := fmt.Errorf("step type: %s does not match any known type", step.StepType) + log.Error(err) + return err + } + if err := stepInstance.Run(ctx); err != nil { + return err + } + return nil +} + +func prepareScriptsEnv() []string { + scripts := []string{} + scripts = append(scripts, "eval $(ssh-agent -s) > /dev/null") + // $HOME/.ssh/id_rsa 为 github 私钥 + scripts = append(scripts, fmt.Sprintf("ssh-add %s/.ssh/id_rsa.github &> /dev/null", config.Home())) + scripts = append(scripts, fmt.Sprintf("rm %s/.ssh/id_rsa.github &> /dev/null", config.Home())) + // $HOME/.ssh/gitlab 为 gitlab 私钥 + scripts = append(scripts, fmt.Sprintf("ssh-add %s/.ssh/id_rsa.gitlab &> /dev/null", config.Home())) + scripts = append(scripts, fmt.Sprintf("rm %s/.ssh/id_rsa.gitlab &> /dev/null", config.Home())) + + return scripts +} + +func handleCmdOutput(pipe io.ReadCloser, needPersistentLog bool, logFile string, secretEnvs []string) { + reader := bufio.NewReader(pipe) + + for { + lineBytes, err := reader.ReadBytes('\n') + if err != nil { + if err == io.EOF { + break + } + + log.Errorf("Failed to read log when processing cmd output: %s", err) + break + } + + fmt.Printf("%s", maskSecretEnvs(string(lineBytes), secretEnvs)) + + if needPersistentLog { + err := util.WriteFile(logFile, lineBytes, 0700) + if err != nil { + log.Warnf("Failed to write file when processing cmd output: %s", err) + } + } + } +} + +const ( + secretEnvMask = "********" +) + +func maskSecret(secrets []string, message string) string { + out := message + + for _, val := range secrets { + if len(val) == 0 { + continue + } + out = strings.Replace(out, val, "********", -1) + } + return out +} + +func maskSecretEnvs(message string, secretEnvs []string) string { + out := message + + for _, val := range secretEnvs { + if len(val) == 0 { + continue + } + sl := strings.Split(val, "=") + + if len(sl) != 2 { + continue + } + + if len(sl[0]) == 0 || len(sl[1]) == 0 { + // invalid key value pair received + continue + } + out = strings.Replace(out, strings.Join(sl[1:], "="), secretEnvMask, -1) + } + return out +} + +func isDirEmpty(dir string) bool { + f, err := os.Open(dir) + if err != nil { + return true + } + defer f.Close() + + _, err = f.Readdir(1) + return err == io.EOF +} + +func setCmdsWorkDir(dir string, cmds []*cmd.Command) { + for _, c := range cmds { + c.Cmd.Dir = dir + } +} diff --git a/pkg/microservice/jobexecutor/core/service/step/step_archive.go b/pkg/microservice/jobexecutor/core/service/step/step_archive.go new file mode 100644 index 0000000000000000000000000000000000000000..c21422b0c16fb5fa9b8d27aeab0cbd9a21c59854 --- /dev/null +++ b/pkg/microservice/jobexecutor/core/service/step/step_archive.go @@ -0,0 +1,126 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package step + +import ( + "context" + "fmt" + "os" + "path" + "path/filepath" + "strings" + "time" + + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/tool/s3" + "github.com/koderover/zadig/pkg/types/step" + "gopkg.in/yaml.v3" +) + +type ArchiveStep struct { + spec *step.StepArchiveSpec + envs []string + secretEnvs []string + workspace string +} + +func NewArchiveStep(spec interface{}, workspace string, envs, secretEnvs []string) (*ArchiveStep, error) { + archiveStep := &ArchiveStep{workspace: workspace, envs: envs, secretEnvs: secretEnvs} + yamlBytes, err := yaml.Marshal(spec) + if err != nil { + return archiveStep, fmt.Errorf("marshal spec %+v failed", spec) + } + if err := yaml.Unmarshal(yamlBytes, &archiveStep.spec); err != nil { + return archiveStep, fmt.Errorf("unmarshal spec %s to shell spec failed", yamlBytes) + } + return archiveStep, nil +} + +func (s *ArchiveStep) Run(ctx context.Context) error { + start := time.Now() + defer func() { + log.Infof("Archive ended. Duration: %.2f seconds", time.Since(start).Seconds()) + }() + + for _, upload := range s.spec.UploadDetail { + log.Infof("Start archive %s.", upload.FilePath) + if upload.DestinationPath == "" || upload.FilePath == "" { + return nil + } + forcedPathStyle := true + if s.spec.S3.Provider == setting.ProviderSourceAli { + forcedPathStyle = false + } + client, err := s3.NewClient(s.spec.S3.Endpoint, s.spec.S3.Ak, s.spec.S3.Sk, s.spec.S3.Insecure, forcedPathStyle) + if err != nil { + return fmt.Errorf("failed to create s3 client to upload file, err: %s", err) + } + + envmaps := make(map[string]string) + for _, env := range s.envs { + kv := strings.Split(env, "=") + if len(kv) != 2 { + continue + } + envmaps[kv[0]] = kv[1] + } + for _, secretEnv := range s.secretEnvs { + kv := strings.Split(secretEnv, "=") + if len(kv) != 2 { + continue + } + envmaps[kv[0]] = kv[1] + } + + upload.AbsFilePath = fmt.Sprintf("$WORKSPACE/%s", upload.FilePath) + upload.AbsFilePath = replaceEnvWithValue(upload.AbsFilePath, envmaps) + upload.DestinationPath = replaceEnvWithValue(upload.DestinationPath, envmaps) + + if len(s.spec.S3.Subfolder) > 0 { + upload.DestinationPath = strings.TrimLeft(path.Join(s.spec.S3.Subfolder, upload.DestinationPath), "/") + } + + info, err := os.Stat(upload.AbsFilePath) + if err != nil { + return fmt.Errorf("failed to upload file path [%s] to destination [%s], the error is: %s", upload.AbsFilePath, upload.DestinationPath, err) + } + // if the given path is a directory + if info.IsDir() { + err := client.UploadDir(s.spec.S3.Bucket, upload.AbsFilePath, upload.DestinationPath) + if err != nil { + return err + } + } else { + key := filepath.Join(upload.DestinationPath, info.Name()) + err := client.Upload(s.spec.S3.Bucket, upload.AbsFilePath, key) + if err != nil { + return err + } + } + } + return nil +} + +func replaceEnvWithValue(str string, envs map[string]string) string { + ret := str + for key, value := range envs { + strKey := fmt.Sprintf("$%s", key) + ret = strings.ReplaceAll(ret, strKey, value) + } + return ret +} diff --git a/pkg/microservice/jobexecutor/core/service/step/step_docker_build.go b/pkg/microservice/jobexecutor/core/service/step/step_docker_build.go new file mode 100644 index 0000000000000000000000000000000000000000..9791c69e5acff873114c2f532afb3b31d9577056 --- /dev/null +++ b/pkg/microservice/jobexecutor/core/service/step/step_docker_build.go @@ -0,0 +1,200 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package step + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "os/exec" + "strings" + "time" + + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types/step" + "github.com/koderover/zadig/pkg/util/fs" + "gopkg.in/yaml.v3" +) + +const dockerExe = "/usr/local/bin/docker" + +type DockerBuildStep struct { + spec *step.StepDockerBuildSpec + envs []string + secretEnvs []string + workspace string +} + +func NewDockerBuildStep(spec interface{}, workspace string, envs, secretEnvs []string) (*DockerBuildStep, error) { + dockerBuildStep := &DockerBuildStep{workspace: workspace, envs: envs, secretEnvs: secretEnvs} + yamlBytes, err := yaml.Marshal(spec) + if err != nil { + return dockerBuildStep, fmt.Errorf("marshal spec %+v failed", spec) + } + if err := yaml.Unmarshal(yamlBytes, &dockerBuildStep.spec); err != nil { + return dockerBuildStep, fmt.Errorf("unmarshal spec %s to docker build spec failed", yamlBytes) + } + return dockerBuildStep, nil +} + +func (s *DockerBuildStep) Run(ctx context.Context) error { + start := time.Now() + log.Infof("Start docker build.") + defer func() { + log.Infof("Docker build ended. Duration: %.2f seconds.", time.Since(start).Seconds()) + }() + + if err := s.dockerLogin(); err != nil { + return err + } + return s.runDockerBuild() +} + +func (s DockerBuildStep) dockerLogin() error { + if s.spec.DockerRegistry == nil { + return nil + } + if s.spec.DockerRegistry.UserName != "" { + fmt.Printf("Logining Docker Registry: %s.\n", s.spec.DockerRegistry.Host) + startTimeDockerLogin := time.Now() + cmd := dockerLogin(s.spec.DockerRegistry.UserName, s.spec.DockerRegistry.Password, s.spec.DockerRegistry.Host) + var out bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &out + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to login docker registry: %s %s", err, out.String()) + } + + fmt.Printf("Login ended. Duration: %.2f seconds.\n", time.Since(startTimeDockerLogin).Seconds()) + } + return nil +} + +func (s *DockerBuildStep) runDockerBuild() error { + if s.spec == nil { + return nil + } + + fmt.Printf("Preparing Dockerfile.\n") + startTimePrepareDockerfile := time.Now() + err := prepareDockerfile(s.spec.Source, s.spec.DockerTemplateContent) + if err != nil { + return fmt.Errorf("failed to prepare dockerfile: %s", err) + } + fmt.Printf("Preparation ended. Duration: %.2f seconds.\n", time.Since(startTimePrepareDockerfile).Seconds()) + + if s.spec.Proxy != nil { + setProxy(s.spec) + } + + fmt.Printf("Runing Docker Build.\n") + startTimeDockerBuild := time.Now() + envs := s.envs + for _, c := range s.dockerCommands() { + c.Stdout = os.Stdout + c.Stderr = os.Stderr + c.Dir = s.workspace + c.Env = envs + if err := c.Run(); err != nil { + return fmt.Errorf("failed to run docker build: %s", err) + } + } + fmt.Printf("Docker build ended. Duration: %.2f seconds.\n", time.Since(startTimeDockerBuild).Seconds()) + + return nil +} + +func (s *DockerBuildStep) dockerCommands() []*exec.Cmd { + cmds := make([]*exec.Cmd, 0) + cmds = append( + cmds, + dockerBuildCmd( + s.spec.GetDockerFile(), + s.spec.ImageName, + s.spec.WorkDir, + s.spec.BuildArgs, + s.spec.IgnoreCache, + ), + dockerPush(s.spec.ImageName), + ) + return cmds +} + +func dockerBuildCmd(dockerfile, fullImage, ctx, buildArgs string, ignoreCache bool) *exec.Cmd { + args := []string{"-c"} + dockerCommand := "docker build --rm=true" + if ignoreCache { + dockerCommand += " --no-cache" + } + + if buildArgs != "" { + for _, val := range strings.Fields(buildArgs) { + if val != "" { + dockerCommand = dockerCommand + " " + val + } + } + + } + dockerCommand = dockerCommand + " -t " + fullImage + " -f " + dockerfile + " " + ctx + args = append(args, dockerCommand) + return exec.Command("sh", args...) +} + +func dockerPush(fullImage string) *exec.Cmd { + args := []string{ + "push", + fullImage, + } + return exec.Command(dockerExe, args...) +} + +func dockerLogin(user, password, registry string) *exec.Cmd { + return exec.Command( + dockerExe, + "login", + "-u", user, + "-p", password, + registry, + ) +} + +func prepareDockerfile(dockerfileSource, dockerfileContent string) error { + if dockerfileSource == setting.DockerfileSourceTemplate { + reader := strings.NewReader(dockerfileContent) + readCloser := io.NopCloser(reader) + path := fmt.Sprintf("/%s", setting.ZadigDockerfilePath) + err := fs.SaveFile(readCloser, path) + if err != nil { + return err + } + } + return nil +} + +func setProxy(ctx *step.StepDockerBuildSpec) { + if ctx.Proxy.EnableRepoProxy && ctx.Proxy.Type == "http" { + if !strings.Contains(strings.ToLower(ctx.BuildArgs), "--build-arg http_proxy=") { + ctx.BuildArgs = fmt.Sprintf("%s --build-arg http_proxy=%s", ctx.BuildArgs, ctx.Proxy.GetProxyURL()) + } + if !strings.Contains(strings.ToLower(ctx.BuildArgs), "--build-arg https_proxy=") { + ctx.BuildArgs = fmt.Sprintf("%s --build-arg https_proxy=%s", ctx.BuildArgs, ctx.Proxy.GetProxyURL()) + } + } +} diff --git a/pkg/microservice/jobexecutor/core/service/step/step_git.go b/pkg/microservice/jobexecutor/core/service/step/step_git.go new file mode 100644 index 0000000000000000000000000000000000000000..f0fdcc12df438e29887468845963a1674708dc21 --- /dev/null +++ b/pkg/microservice/jobexecutor/core/service/step/step_git.go @@ -0,0 +1,378 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package step + +import ( + "bufio" + "context" + "encoding/base64" + "fmt" + "io/ioutil" + "net/url" + "os" + "path" + "path/filepath" + "strings" + "time" + + "gopkg.in/yaml.v3" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/koderover/zadig/pkg/microservice/jobexecutor/config" + c "github.com/koderover/zadig/pkg/microservice/jobexecutor/core/service/cmd" + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types" + "github.com/koderover/zadig/pkg/types/step" +) + +type GitStep struct { + spec *step.StepGitSpec + envs []string + secretEnvs []string + workspace string +} + +func NewGitStep(spec interface{}, workspace string, envs, secretEnvs []string) (*GitStep, error) { + gitStep := &GitStep{workspace: workspace, envs: envs, secretEnvs: secretEnvs} + yamlBytes, err := yaml.Marshal(spec) + if err != nil { + return gitStep, fmt.Errorf("marshal spec %+v failed", spec) + } + if err := yaml.Unmarshal(yamlBytes, &gitStep.spec); err != nil { + return gitStep, fmt.Errorf("unmarshal spec %s to shell spec failed", yamlBytes) + } + return gitStep, nil +} + +func (s *GitStep) Run(ctx context.Context) error { + start := time.Now() + log.Infof("Start git clone.") + defer func() { + log.Infof("Git clone ended. Duration: %.2f seconds.", time.Since(start).Seconds()) + }() + return s.runGitCmds() +} + +func (s *GitStep) RunGitGc(folder string) error { + // envs := r.getUserEnvs() + cmd := c.Gc() + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + // cmd.Env = envs + cmd.Dir = path.Join(s.workspace, folder) + return cmd.Run() +} + +func (s *GitStep) runGitCmds() error { + + envs := s.envs + // 如果存在github代码库，则设置代理，同时保证非github库不走代理 + if s.spec.Proxy != nil && s.spec.Proxy.EnableRepoProxy && s.spec.Proxy.Type == "http" { + noProxy := "" + proxyFlag := false + for _, repo := range s.spec.Repos { + if repo.EnableProxy { + if !proxyFlag { + envs = append(envs, fmt.Sprintf("http_proxy=%s", s.spec.Proxy.GetProxyURL())) + envs = append(envs, fmt.Sprintf("https_proxy=%s", s.spec.Proxy.GetProxyURL())) + proxyFlag = true + } + } else { + uri, err := url.Parse(repo.Address) + if err == nil { + if noProxy != "" { + noProxy += "," + } + noProxy += uri.Host + } + } + } + envs = append(envs, fmt.Sprintf("no_proxy=%s", noProxy)) + } + + // 获取git代码 + cmds := make([]*c.Command, 0) + + cmds = append(cmds, &c.Command{Cmd: c.SetConfig("user.email", "ci@koderover.com"), DisableTrace: true}) + cmds = append(cmds, &c.Command{Cmd: c.SetConfig("user.name", "koderover"), DisableTrace: true}) + + // https://stackoverflow.com/questions/24952683/git-push-error-rpc-failed-result-56-http-code-200-fatal-the-remote-end-hun/36843260 + //cmds = append(cmds, &c.Command{Cmd: c.SetConfig("http.postBuffer", "524288000"), DisableTrace: true}) + cmds = append(cmds, &c.Command{Cmd: c.SetConfig("http.postBuffer", "2097152000"), DisableTrace: true}) + var tokens []string + var hostNames = sets.NewString() + for _, repo := range s.spec.Repos { + if repo == nil || len(repo.RepoName) == 0 { + continue + } + + if repo.Source == types.ProviderGerrit { + userpass, _ := base64.StdEncoding.DecodeString(repo.OauthToken) + userpassPair := strings.Split(string(userpass), ":") + var user, password string + if len(userpassPair) > 1 { + password = userpassPair[1] + } + user = userpassPair[0] + repo.Username = user + if password != "" { + repo.Password = password + tokens = append(tokens, repo.Password) + } + } else if repo.Source == types.ProviderCodehub { + tokens = append(tokens, repo.Password) + } else if repo.Source == types.ProviderOther { + tokens = append(tokens, repo.PrivateAccessToken) + tokens = append(tokens, repo.SSHKey) + } + tokens = append(tokens, repo.OauthToken) + cmds = append(cmds, s.buildGitCommands(repo, hostNames)...) + } + // write ssh key + if len(hostNames.List()) > 0 { + if err := writeSSHConfigFile(hostNames, s.spec.Proxy); err != nil { + return err + } + } + + for _, c := range cmds { + cmdOutReader, err := c.Cmd.StdoutPipe() + if err != nil { + return err + } + + outScanner := bufio.NewScanner(cmdOutReader) + go func() { + for outScanner.Scan() { + fmt.Printf("%s\n", maskSecret(tokens, outScanner.Text())) + } + }() + + cmdErrReader, err := c.Cmd.StderrPipe() + if err != nil { + return err + } + + errScanner := bufio.NewScanner(cmdErrReader) + go func() { + for errScanner.Scan() { + fmt.Printf("%s\n", maskSecret(tokens, errScanner.Text())) + } + }() + + c.Cmd.Env = envs + if !c.DisableTrace { + fmt.Printf("%s\n", strings.Join(c.Cmd.Args, " ")) + } + if err := c.Cmd.Run(); err != nil { + if c.IgnoreError { + continue + } + return err + } + } + return nil +} + +func (s *GitStep) buildGitCommands(repo *types.Repository, hostNames sets.String) []*c.Command { + + cmds := make([]*c.Command, 0) + + if len(repo.RepoName) == 0 { + return cmds + } + + workDir := filepath.Join(s.workspace, repo.RepoName) + if len(repo.CheckoutPath) != 0 { + workDir = filepath.Join(s.workspace, repo.CheckoutPath) + } + + if _, err := os.Stat(workDir); os.IsNotExist(err) { + os.MkdirAll(workDir, 0777) + } + + // 预防非正常退出导致git被锁住 + indexLockPath := path.Join(workDir, "/.git/index.lock") + if err := os.RemoveAll(indexLockPath); err != nil { + log.Errorf("Failed to remove %s: %s", indexLockPath, err) + } + shallowLockPath := path.Join(workDir, "/.git/shallow.lock") + if err := os.RemoveAll(shallowLockPath); err != nil { + log.Errorf("Failed to remove %s: %s", shallowLockPath, err) + } + + if isDirEmpty(filepath.Join(workDir, ".git")) { + cmds = append(cmds, &c.Command{Cmd: c.InitGit(workDir)}) + } else { + cmds = append(cmds, &c.Command{Cmd: c.RemoteRemove(repo.RemoteName), DisableTrace: true, IgnoreError: true}) + } + + // namespace represents the real owner + owner := repo.RepoNamespace + if len(owner) == 0 { + owner = repo.RepoOwner + } + if repo.Source == types.ProviderGitlab { + u, _ := url.Parse(repo.Address) + cmds = append(cmds, &c.Command{ + Cmd: c.RemoteAdd(repo.RemoteName, OAuthCloneURL(repo.Source, repo.OauthToken, u.Host, owner, repo.RepoName, u.Scheme)), + DisableTrace: true, + }) + } else if repo.Source == types.ProviderGerrit { + u, _ := url.Parse(repo.Address) + u.Path = fmt.Sprintf("/a/%s", repo.RepoName) + u.User = url.UserPassword(repo.Username, repo.Password) + + cmds = append(cmds, &c.Command{ + Cmd: c.RemoteAdd(repo.RemoteName, u.String()), + DisableTrace: true, + }) + } else if repo.Source == types.ProviderCodehub { + u, _ := url.Parse(repo.Address) + user := url.QueryEscape(repo.Username) + cmds = append(cmds, &c.Command{ + Cmd: c.RemoteAdd(repo.RemoteName, fmt.Sprintf("%s://%s:%s@%s/%s/%s.git", u.Scheme, user, repo.Password, u.Host, owner, repo.RepoName)), + DisableTrace: true, + }) + } else if repo.Source == types.ProviderGitee { + cmds = append(cmds, &c.Command{Cmd: c.RemoteAdd(repo.RemoteName, HTTPSCloneURL(repo.Source, repo.OauthToken, repo.RepoOwner, repo.RepoName)), DisableTrace: true}) + } else if repo.Source == types.ProviderOther { + if repo.AuthType == types.SSHAuthType { + host := getHost(repo.Address) + if !hostNames.Has(host) { + if err := writeSSHFile(repo.SSHKey, host); err != nil { + log.Errorf("failed to write ssh file %s: %s", repo.SSHKey, err) + } + hostNames.Insert(host) + } + remoteName := fmt.Sprintf("%s:%s/%s.git", repo.Address, repo.RepoOwner, repo.RepoName) + // Including the case of the port + if strings.Contains(repo.Address, ":") { + remoteName = fmt.Sprintf("%s/%s/%s.git", repo.Address, repo.RepoOwner, repo.RepoName) + } + cmds = append(cmds, &c.Command{ + Cmd: c.RemoteAdd(repo.RemoteName, remoteName), + DisableTrace: true, + }) + } else if repo.AuthType == types.PrivateAccessTokenAuthType { + u, err := url.Parse(repo.Address) + if err != nil { + log.Errorf("failed to parse url,err:%s", err) + } else { + cmds = append(cmds, &c.Command{ + Cmd: c.RemoteAdd(repo.RemoteName, OAuthCloneURL(repo.Source, repo.PrivateAccessToken, u.Host, repo.RepoOwner, repo.RepoName, u.Scheme)), + DisableTrace: true, + }) + } + } + } else { + // github + cmds = append(cmds, &c.Command{Cmd: c.RemoteAdd(repo.RemoteName, HTTPSCloneURL(repo.Source, repo.OauthToken, owner, repo.RepoName)), DisableTrace: true}) + } + + ref := repo.Ref() + if ref == "" { + return cmds + } + + cmds = append(cmds, &c.Command{Cmd: c.Fetch(repo.RemoteName, ref)}, &c.Command{Cmd: c.CheckoutHead()}) + + // PR rebase branch 请求 + if repo.PR > 0 && len(repo.Branch) > 0 { + newBranch := fmt.Sprintf("pr%d", repo.PR) + ref := fmt.Sprintf("%s:%s", repo.PRRef(), newBranch) + cmds = append( + cmds, + &c.Command{Cmd: c.DeepenedFetch(repo.RemoteName, repo.BranchRef())}, + &c.Command{Cmd: c.ResetMerge()}, + &c.Command{Cmd: c.DeepenedFetch(repo.RemoteName, ref)}, + &c.Command{Cmd: c.Merge(newBranch)}, + ) + } + + if repo.SubModules { + cmds = append(cmds, &c.Command{Cmd: c.UpdateSubmodules()}) + } + + cmds = append(cmds, &c.Command{Cmd: c.ShowLastLog()}) + + setCmdsWorkDir(workDir, cmds) + + return cmds +} + +func writeSSHFile(sshKey, hostName string) error { + if sshKey == "" { + return fmt.Errorf("ssh cannot be empty") + } + + if hostName == "" { + return fmt.Errorf("hostName cannot be empty") + } + + hostName = strings.Replace(hostName, ".", "", -1) + hostName = strings.Replace(hostName, ":", "", -1) + pathName := fmt.Sprintf("/.ssh/id_rsa.%s", hostName) + file := path.Join(config.Home(), pathName) + return ioutil.WriteFile(file, []byte(sshKey), 0400) +} + +func writeSSHConfigFile(hostNames sets.String, proxy *step.Proxy) error { + out := "\nHOST *\nStrictHostKeyChecking=no\nUserKnownHostsFile=/dev/null\n" + for _, hostName := range hostNames.List() { + name := strings.Replace(hostName, ".", "", -1) + name = strings.Replace(name, ":", "", -1) + out += fmt.Sprintf("\nHost %s\nIdentityFile ~/.ssh/id_rsa.%s\n", hostName, name) + if proxy.EnableRepoProxy && proxy.Type == "socks5" { + out = out + fmt.Sprintf("ProxyCommand nc -x %s %%h %%p\n", proxy.GetProxyURL()) + } + } + file := path.Join(config.Home(), "/.ssh/config") + return ioutil.WriteFile(file, []byte(out), 0600) +} + +// git@github.com or git@github.com:2000 +// return github.com +func getHost(address string) string { + address = strings.TrimPrefix(address, "ssh://") + address = strings.TrimPrefix(address, "git@") + hostArr := strings.Split(address, ":") + return hostArr[0] +} + +// SSHCloneURL returns Oauth clone url +// e.g. +//https://oauth2:ACCESS_TOKEN@somegitlab.com/owner/name.git +func OAuthCloneURL(source, token, address, owner, name, scheme string) string { + if strings.ToLower(source) == types.ProviderGitlab || strings.ToLower(source) == types.ProviderOther { + // address 需要传过来 + return fmt.Sprintf("%s://%s:%s@%s/%s/%s.git", scheme, step.OauthTokenPrefix, token, address, owner, name) + } + // GITHUB + return "github" +} + +// HTTPSCloneURL returns HTTPS clone url +func HTTPSCloneURL(source, token, owner, name string) string { + if strings.ToLower(source) == types.ProviderGitlab { + return fmt.Sprintf("https://%s/%s/%s.git", "gitlab.koderover.com", owner, name) + } else if strings.ToLower(source) == types.ProviderGitee { + return fmt.Sprintf("https://%s:%s@%s/%s/%s.git", step.OauthTokenPrefix, token, "gitee.com", owner, name) + } + //return fmt.Sprintf("https://x-access-token:%s@%s/%s/%s.git", g.GetInstallationToken(owner), g.GetGithubHost(), owner, name) + return fmt.Sprintf("https://x-access-token:%s@%s/%s/%s.git", token, "github.com", owner, name) +} diff --git a/pkg/microservice/jobexecutor/core/service/step/step_shell.go b/pkg/microservice/jobexecutor/core/service/step/step_shell.go new file mode 100644 index 0000000000000000000000000000000000000000..fa679e36d9a9e6a3f1cd451eb1bef56ae7674919 --- /dev/null +++ b/pkg/microservice/jobexecutor/core/service/step/step_shell.go @@ -0,0 +1,118 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package step + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types/step" + "github.com/koderover/zadig/pkg/util" + "gopkg.in/yaml.v3" +) + +type ShellStep struct { + spec *step.StepShellSpec + envs []string + secretEnvs []string + workspace string + Paths string +} + +func NewShellStep(spec interface{}, workspace, paths string, envs, secretEnvs []string) (*ShellStep, error) { + shellStep := &ShellStep{workspace: workspace, envs: envs, secretEnvs: secretEnvs} + yamlBytes, err := yaml.Marshal(spec) + if err != nil { + return shellStep, fmt.Errorf("marshal spec %+v failed", spec) + } + if err := yaml.Unmarshal(yamlBytes, &shellStep.spec); err != nil { + return shellStep, fmt.Errorf("unmarshal spec %s to shell spec failed", yamlBytes) + } + return shellStep, nil +} + +func (s *ShellStep) Run(ctx context.Context) error { + start := time.Now() + log.Infof("Executing user script.") + defer func() { + log.Infof("Script Execution ended. Duration: %.2f seconds.", time.Since(start).Seconds()) + }() + + if len(s.spec.Scripts) == 0 { + return nil + } + scripts := prepareScriptsEnv() + + scripts = append(scripts, s.spec.Scripts...) + + userScriptFile := "user_script.sh" + if err := ioutil.WriteFile(filepath.Join(os.TempDir(), userScriptFile), []byte(strings.Join(scripts, "\n")), 0700); err != nil { + return fmt.Errorf("write script file error: %v", err) + } + + cmd := exec.Command("/bin/bash", filepath.Join(os.TempDir(), userScriptFile)) + cmd.Dir = s.workspace + cmd.Env = s.envs + + fileName := filepath.Join(os.TempDir(), "user_script.log") + //如果文件不存在就创建文件，避免后面使用变量出错 + util.WriteFile(fileName, []byte{}, 0700) + + needPersistentLog := true + + var wg sync.WaitGroup + + cmdStdoutReader, err := cmd.StdoutPipe() + if err != nil { + return err + } + + wg.Add(1) + go func() { + defer wg.Done() + + handleCmdOutput(cmdStdoutReader, needPersistentLog, fileName, s.secretEnvs) + }() + + cmdStdErrReader, err := cmd.StderrPipe() + if err != nil { + return err + } + + wg.Add(1) + go func() { + defer wg.Done() + + handleCmdOutput(cmdStdErrReader, needPersistentLog, fileName, s.secretEnvs) + }() + + if err := cmd.Start(); err != nil { + return err + } + + wg.Wait() + + return cmd.Wait() +} diff --git a/pkg/microservice/jobexecutor/core/service/step/step_tool_install.go b/pkg/microservice/jobexecutor/core/service/step/step_tool_install.go new file mode 100644 index 0000000000000000000000000000000000000000..c341e026c69ca99782375a98d6f67502f6d31517 --- /dev/null +++ b/pkg/microservice/jobexecutor/core/service/step/step_tool_install.go @@ -0,0 +1,175 @@ +package step + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "time" + + "github.com/google/uuid" + "github.com/koderover/zadig/pkg/microservice/reaper/config" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/httpclient" + "github.com/koderover/zadig/pkg/tool/log" + + s3tool "github.com/koderover/zadig/pkg/tool/s3" + "github.com/koderover/zadig/pkg/types/step" + "gopkg.in/yaml.v3" +) + +type ToolInstallStep struct { + spec *step.StepToolInstallSpec + envs []string + secretEnvs []string + workspace string +} + +func NewToolInstallStep(spec interface{}, workspace string, envs, secretEnvs []string) (*ToolInstallStep, error) { + toolInstallStep := &ToolInstallStep{workspace: workspace, envs: envs, secretEnvs: secretEnvs} + yamlBytes, err := yaml.Marshal(spec) + if err != nil { + return toolInstallStep, fmt.Errorf("marshal spec %+v failed", spec) + } + if err := yaml.Unmarshal(yamlBytes, &toolInstallStep.spec); err != nil { + return toolInstallStep, fmt.Errorf("unmarshal spec %s to shell spec failed", yamlBytes) + } + return toolInstallStep, nil +} + +func (s *ToolInstallStep) Run(ctx context.Context) error { + start := time.Now() + log.Infof("Installing tools.") + defer func() { + log.Infof("Install tools ended. Duration: %.2f seconds.", time.Since(start).Seconds()) + }() + + for _, tool := range s.spec.Installs { + log.Infof("Installing %s %s.", tool.Name, tool.Version) + if err := s.runIntallationScripts(tool); err != nil { + return err + } + } + + return nil +} + +func (s *ToolInstallStep) runIntallationScripts(tool *step.Tool) error { + if tool == nil { + return nil + } + var ( + openProxy bool + proxyScript, disProxyScript string + ) + + var tmpPath string + scripts := []string{} + scripts = append(scripts, "set -ex") + + // 获取用户指定环境变量 + s.envs = append(s.envs, Environs(tool.Envs)...) + + if openProxy { + scripts = append(scripts, proxyScript) + } + + // 如果应用有配置下载路径 + if tool.Download != "" { + s.spec.S3Storage.Subfolder = fmt.Sprintf("%s/%s-v%s", config.ConstructCachePath, tool.Name, tool.Version) + filepath := strings.Split(tool.Download, "/") + fileName := filepath[len(filepath)-1] + forcedPathStyle := true + if s.spec.S3Storage.Provider == setting.ProviderSourceAli { + forcedPathStyle = false + } + + tmpPath = path.Join(os.TempDir(), fileName) + s3client, err := s3tool.NewClient(s.spec.S3Storage.Endpoint, s.spec.S3Storage.Ak, s.spec.S3Storage.Sk, s.spec.S3Storage.Insecure, forcedPathStyle) + if err == nil { + objectKey := GetObjectPath(fileName, s.spec.S3Storage.Subfolder) + err = s3client.Download( + s.spec.S3Storage.Bucket, + objectKey, + tmpPath, + ) + + // 缓存不存在 + if err != nil { + err := httpclient.Download(tool.Download, tmpPath) + if err != nil { + return err + } + s3client.Upload( + s.spec.S3Storage.Bucket, + tmpPath, + objectKey, + ) + fmt.Printf("Package loaded from url: %s\n", tool.Download) + } + } else { + err := httpclient.Download(tool.Download, tmpPath) + if err != nil { + return err + } + } + } + + for j, command := range tool.Scripts { + realCommand := strings.ReplaceAll(command, config.FilepathParam, tmpPath) + tool.Scripts[j] = realCommand + } + + scripts = append(scripts, tool.Scripts...) + + if openProxy { + scripts = append(scripts, disProxyScript) + } + uid, _ := uuid.NewUUID() + file := filepath.Join(os.TempDir(), fmt.Sprintf("install_script_%d.sh", uid)) + if err := ioutil.WriteFile(file, []byte(strings.Join(scripts, "\n")), 0700); err != nil { + return fmt.Errorf("write script file error: %v", err) + } + + cmd := exec.Command("/bin/bash", file) + cmd.Dir = s.workspace + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Env = s.envs + + if err := cmd.Run(); err != nil { + return err + } + + return nil +} + +func Environs(envs []string) []string { + resp := []string{} + for _, val := range envs { + if val == "" { + continue + } + + if len(strings.Split(val, "=")) != 2 { + continue + } + + replaced := strings.Replace(val, "$HOME", config.Home(), -1) + resp = append(resp, replaced) + } + return resp +} + +func GetObjectPath(name, subFolder string) string { + // target should not be started with / + if subFolder != "" { + return strings.TrimLeft(filepath.Join(subFolder, name), "/") + } + + return strings.TrimLeft(name, "/") +} diff --git a/pkg/microservice/jobexecutor/executor/executor.go b/pkg/microservice/jobexecutor/executor/executor.go new file mode 100644 index 0000000000000000000000000000000000000000..b3bdd9acd89532b1ee8b841858bca75f6a164a77 --- /dev/null +++ b/pkg/microservice/jobexecutor/executor/executor.go @@ -0,0 +1,90 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package executor + +import ( + "context" + "fmt" + "io/ioutil" + "time" + + commonconfig "github.com/koderover/zadig/pkg/config" + job "github.com/koderover/zadig/pkg/microservice/jobexecutor/core/service" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types" +) + +// const ( +// ZadigContextDir = "/zadig/" +// ZadigLogFile = ZadigContextDir + "zadig.log" +// ZadigLifeCycleFile = ZadigContextDir + "lifecycle" +// ) + +func Execute(ctx context.Context) error { + // f, err := os.OpenFile(ZadigLogFile, os.O_WRONLY|os.O_CREATE|os.O_SYNC|os.O_APPEND, 0755) + // if err != nil { + // fmt.Println("open log file error") + // return err + // } + // os.Stdout = f + // os.Stderr = f + + log.Init(&log.Config{ + Level: commonconfig.LogLevel(), + NoCaller: true, + NoLogLevel: true, + Development: commonconfig.Mode() != setting.ReleaseMode, + // SendToFile: true, + // Filename: ZadigLogFile, + }) + + start := time.Now() + + excutor := "job-executor" + var err error + defer func() { + // os.Remove(ZadigLifeCycleFile) + resultMsg := types.JobSuccess + if err != nil { + resultMsg = types.JobFail + fmt.Printf("Failed to run: %s.\n", err) + } + fmt.Printf("Job Status: %s\n", resultMsg) + dogFoodErr := ioutil.WriteFile(setting.DogFood, []byte(resultMsg), 0644) + if dogFoodErr != nil { + log.Errorf("Failed to create dog food: %s.", dogFoodErr) + } + + fmt.Printf("====================== %s End. Duration: %.2f seconds ======================\n", excutor, time.Since(start).Seconds()) + time.Sleep(30 * time.Second) + }() + + var j *job.Job + j, err = job.NewJob() + if err != nil { + return err + } + fmt.Printf("====================== %s Start ======================\n", excutor) + if err = j.Run(ctx); err != nil { + return err + } + if err = j.AfterRun(ctx); err != nil { + return err + } + return nil +} diff --git a/pkg/tool/kube/updater/clusterrolebinding.go b/pkg/microservice/packager/config/config.go similarity index 65% rename from pkg/tool/kube/updater/clusterrolebinding.go rename to pkg/microservice/packager/config/config.go index 7e3f2fafc6f3a586a3f756121375b86d022a8d38..76ed9e11392ee3c6a6fdffcddf2fb3a07bc4474e 100644 --- a/pkg/tool/kube/updater/clusterrolebinding.go +++ b/pkg/microservice/packager/config/config.go @@ -14,14 +14,20 @@ See the License for the specific language governing permissions and limitations under the License. */ -package updater +package config import ( - rbacv1beta1 "k8s.io/api/rbac/v1beta1" - "k8s.io/apimachinery/pkg/labels" - "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/spf13/viper" + + // init the config first + _ "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/setting" ) -func DeleteClusterRoleBindings(selector labels.Selector, cl client.Client) error { - return deleteObjectsWithDefaultOptions("", selector, &rbacv1beta1.ClusterRoleBinding{}, cl) +func JobConfigFile() string { + return viper.GetString(setting.JobConfigFile) +} + +func Home() string { + return viper.GetString(setting.Home) } diff --git a/pkg/microservice/packager/core/service/packager.go b/pkg/microservice/packager/core/service/packager.go new file mode 100644 index 0000000000000000000000000000000000000000..7ee5ff44326de4515d23723da21effdc9daa32ad --- /dev/null +++ b/pkg/microservice/packager/core/service/packager.go @@ -0,0 +1,49 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +type ImageData struct { + ImageUrl string `yaml:"image_url" json:"image_url"` + ImageName string `yaml:"image_name" json:"image_name"` + ImageTag string `yaml:"image_tag" json:"image_tag"` + CustomTag string `yaml:"custom_tag" json:"custom_tag"` + RegistryID string `yaml:"registry_id" json:"registry_id"` +} + +// ImagesByService defines all images in a service +type ImagesByService struct { + ServiceName string `yaml:"service_name"` + Images []*ImageData `yaml:"images"` +} + +//DockerRegistry registry host/user/password +type DockerRegistry struct { + RegistryID string `yaml:"registry_id"` + Host string `yaml:"host"` + UserName string `yaml:"username"` + Password string `yaml:"password"` + Namespace string `yaml:"namespace"` +} + +// Context parameters for job to run with +type Context struct { + JobType string `yaml:"job_type"` + ProgressFile string `yaml:"progress_file"` + Images []*ImagesByService `yaml:"images"` + SourceRegistries []*DockerRegistry `yaml:"source_registries"` + TargetRegistries []*DockerRegistry `yaml:"target_registries"` +} diff --git a/pkg/microservice/packager/core/service/service.go b/pkg/microservice/packager/core/service/service.go new file mode 100644 index 0000000000000000000000000000000000000000..fb9c958e4fa39cf09256f4216fe0cc476577377c --- /dev/null +++ b/pkg/microservice/packager/core/service/service.go @@ -0,0 +1,288 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" + "github.com/docker/docker/pkg/jsonmessage" + "github.com/pkg/errors" + "gopkg.in/yaml.v3" + + "github.com/koderover/zadig/pkg/microservice/packager/config" + "github.com/koderover/zadig/pkg/tool/log" +) + +type Packager struct { + Ctx *Context +} + +type PackageResult struct { + ServiceName string `json:"service_name"` + Result string `json:"result"` + ErrorMsg string `json:"error_msg"` + ImageData []*ImageData `json:"image_data"` +} + +func NewPackager() (*Packager, error) { + contextData, err := os.ReadFile(config.JobConfigFile()) + if err != nil { + return nil, err + } + + var ctx *Context + if err := yaml.Unmarshal(contextData, &ctx); err != nil { + return nil, err + } + + packager := &Packager{ + Ctx: ctx, + } + + return packager, nil +} + +func buildRegistryMap(registries []*DockerRegistry) map[string]*DockerRegistry { + ret := make(map[string]*DockerRegistry) + for _, registry := range registries { + ret[registry.RegistryID] = registry + } + return ret +} + +func base64EncodeAuth(auth *types.AuthConfig) (string, error) { + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(auth); err != nil { + return "", err + } + return base64.URLEncoding.EncodeToString(buf.Bytes()), nil +} + +func buildTargetImage(imageName, imageTag, host, nameSpace string) string { + ret := "" + if len(nameSpace) > 0 { + ret = fmt.Sprintf("%s/%s/%s:%s", host, nameSpace, imageName, imageTag) + } else { + ret = fmt.Sprintf("%s/%s:%s", host, imageName, imageTag) + } + ret = strings.TrimPrefix(ret, "http://") + ret = strings.TrimPrefix(ret, "https://") + return ret +} + +func ExtractErrorDetail(in io.Reader) error { + dec := json.NewDecoder(in) + for dec.More() { + var jm jsonmessage.JSONMessage + if err := dec.Decode(&jm); err != nil { + return err + } + if jm.Error != nil { + return jm.Error + } + } + return nil +} + +func pullImage(dockerClient *client.Client, imageUrl string, options *types.ImagePullOptions) error { + log.Infof("pulling image: %s", imageUrl) + pullResponse, err := dockerClient.ImagePull(context.TODO(), imageUrl, *options) + if err != nil { + return err + } + + defer pullResponse.Close() + + err = ExtractErrorDetail(pullResponse) + return err +} + +func pushImage(dockerClient *client.Client, targetImageUrl string, options *types.ImagePushOptions) error { + log.Infof("pushing image: %s", targetImageUrl) + pushResponse, err := dockerClient.ImagePush(context.TODO(), targetImageUrl, *options) + if err != nil { + return errors.Wrapf(err, "failed to push image: %s", targetImageUrl) + } + + defer pushResponse.Close() + + err = ExtractErrorDetail(pushResponse) + return err +} + +func handleSingleService(imageByService *ImagesByService, allRegistries map[string]*DockerRegistry, targetRegistries []*DockerRegistry, dockerClient *client.Client) ([]*ImageData, error) { + targetImageUrlByRepo := make(map[string][]string) + retImages := make([]*ImageData, 0) + + for _, singleImage := range imageByService.Images { + options := types.ImagePullOptions{} + // for images from public repo，registryID won't be appointed + if len(singleImage.RegistryID) > 0 { + registryInfo, ok := allRegistries[singleImage.RegistryID] + if !ok { + return nil, fmt.Errorf("failed to find source registry for image: %s", singleImage.ImageUrl) + } + encodedAuth, err := base64EncodeAuth(&types.AuthConfig{ + Username: registryInfo.UserName, + Password: registryInfo.Password, + ServerAddress: registryInfo.Host, + }) + if err != nil { + return nil, errors.Wrapf(err, "faied to create docker pull auth data") + } + options.RegistryAuth = encodedAuth + } + + // pull image + err := pullImage(dockerClient, singleImage.ImageUrl, &options) + if err != nil { + return nil, errors.Wrapf(err, "failed to pull image: %s", singleImage.ImageUrl) + } + + // tag image + for _, registry := range targetRegistries { + targetImage := buildTargetImage(singleImage.ImageName, singleImage.CustomTag, registry.Host, registry.Namespace) + err = dockerClient.ImageTag(context.TODO(), singleImage.ImageUrl, targetImage) + if err != nil { + return nil, errors.Wrapf(err, "failed to tag image from: %s to: %s", singleImage.ImageUrl, targetImage) + } + targetImageUrlByRepo[registry.RegistryID] = append(targetImageUrlByRepo[registry.RegistryID], targetImage) + retImages = append(retImages, &ImageData{ + ImageUrl: targetImage, + ImageName: singleImage.ImageName, + ImageTag: singleImage.ImageTag, + CustomTag: singleImage.CustomTag, + RegistryID: singleImage.RegistryID, + }) + } + } + + // push image + for _, registry := range targetRegistries { + encodedAuth, err := base64EncodeAuth(&types.AuthConfig{ + Username: registry.UserName, + Password: registry.Password, + ServerAddress: registry.Host, + }) + if err != nil { + return nil, errors.Wrapf(err, "faied to create docker push auth data") + } + options := types.ImagePushOptions{ + RegistryAuth: encodedAuth, + } + + for _, targetImageUrl := range targetImageUrlByRepo[registry.RegistryID] { + err = pushImage(dockerClient, targetImageUrl, &options) + if err != nil { + return nil, errors.Wrapf(err, "failed to push image: %s", targetImageUrl) + } + } + } + return retImages, nil +} + +func (p *Packager) Exec() error { + + // init docker client + cli, err := client.NewClientWithOpts(client.FromEnv) + if err != nil { + log.Errorf("failed to init docker client %s", err) + return err + } + + defer func() { + err = cli.Close() + if err != nil { + log.Errorf("failed to close client: %s", err) + } + }() + + // run docker info to ensure docker daemon connection + for i := 0; i < 120; i++ { + _, err := cli.Info(context.TODO()) + if err == nil { + break + } + log.Errorf("failed tor run docker info, try index: %d, err: %s", i, err) + time.Sleep(time.Second * 1) + } + + // create log file + if len(p.Ctx.ProgressFile) >= 0 { + if err := os.MkdirAll(filepath.Dir(p.Ctx.ProgressFile), 0770); err != nil { + return errors.Wrapf(err, "failed to create progress file dir") + } + file, err := os.Create(p.Ctx.ProgressFile) + if err != nil { + return errors.Wrapf(err, "failed to create progress file") + } + if err = file.Close(); err != nil { + return errors.Wrapf(err, "failed to close progress file") + } + } + + allRegistries := append(p.Ctx.SourceRegistries, p.Ctx.TargetRegistries...) + realTimeProgress := make([]*PackageResult, 0) + + for _, imageByService := range p.Ctx.Images { + result := &PackageResult{ + ServiceName: imageByService.ServiceName, + } + images, err := handleSingleService(imageByService, buildRegistryMap(allRegistries), p.Ctx.TargetRegistries, cli) + + if err != nil { + result.Result = "failed" + result.ErrorMsg = err.Error() + log.Errorf("[result][fail][%s][%s]", imageByService.ServiceName, err) + } else { + result.Result = "success" + result.ImageData = images + log.Infof("[result][success][%s]", imageByService.ServiceName) + } + + realTimeProgress = append(realTimeProgress, result) + + if len(p.Ctx.ProgressFile) > 0 { + bs, err := json.Marshal(realTimeProgress) + if err != nil { + log.Errorf("failed to marshal progress data %s", err) + continue + } + err = os.WriteFile(p.Ctx.ProgressFile, bs, 0644) + if err != nil { + log.Errorf("failed to write progress data %s", err) + continue + } + } + } + + // keep job alive for extra 10 seconds to make the runner be able to catch all progress info + // TODO need optimize + time.Sleep(time.Second * 10) + return nil +} diff --git a/pkg/microservice/cron/core/service/client/workflow_test.go b/pkg/microservice/packager/executor/executor.go similarity index 53% rename from pkg/microservice/cron/core/service/client/workflow_test.go rename to pkg/microservice/packager/executor/executor.go index 434dfc7a98e8f7dafafe0f435c643d9edd670388..840ca0134d8973b543237c7f4310a166acf3753a 100644 --- a/pkg/microservice/cron/core/service/client/workflow_test.go +++ b/pkg/microservice/packager/executor/executor.go @@ -14,29 +14,33 @@ See the License for the specific language governing permissions and limitations under the License. */ -package client +package executor import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - + commonconfig "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/packager/core/service" "github.com/koderover/zadig/pkg/setting" "github.com/koderover/zadig/pkg/tool/log" ) -const ( - testAPIHost = "os.koderover.com" - testAPIToken = "token" -) - -func TestGenListWorkFlowReq(t *testing.T) { - client := NewAslanClient( - testAPIHost, - testAPIToken, - ) - req, _ := client.genListWorkFlowReq(log.NopSugaredLogger()) - assert.Equal(t, fmt.Sprintf("%s/workflow/workflow", testAPIHost), req.URL.Path) - assert.Equal(t, fmt.Sprintf("%s %s", setting.RootAPIKey, testAPIToken), req.Header.Get(setting.AuthorizationHeader)) +func Execute() error { + log.Init(&log.Config{ + Level: commonconfig.LogLevel(), + NoCaller: true, + NoLogLevel: true, + Development: commonconfig.Mode() != setting.ReleaseMode, + }) + + packager, err := service.NewPackager() + if err != nil { + log.Errorf("Failed to start packager, error: %s", err) + return err + } + + if err := packager.Exec(); err != nil { + log.Errorf("Failed to run packager, error: %s", err) + return err + } + + return nil } diff --git a/pkg/microservice/picket/client/aslan/artifact.go b/pkg/microservice/picket/client/aslan/artifact.go new file mode 100644 index 0000000000000000000000000000000000000000..abb5a36b214c2740f3ad565ae034d28c3a237b4e --- /dev/null +++ b/pkg/microservice/picket/client/aslan/artifact.go @@ -0,0 +1,47 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aslan + +import ( + "fmt" + "net/http" + "net/url" + + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +func (c *Client) GetArtifactByImage(header http.Header, qs url.Values, image string) ([]byte, error) { + url := fmt.Sprintf("/delivery/artifacts/image") + + res, err := c.Get(url, httpclient.SetHeadersFromHTTPHeader(header), httpclient.SetQueryParamsFromValues(qs), httpclient.SetQueryParam(image, image)) + if err != nil { + return nil, err + } + + return res.Body(), nil +} + +func (c *Client) GetArtifactInfo(header http.Header, qs url.Values, id string) ([]byte, error) { + url := fmt.Sprintf("/delivery/artifacts/%s", id) + + res, err := c.Get(url, httpclient.SetHeadersFromHTTPHeader(header), httpclient.SetQueryParamsFromValues(qs)) + if err != nil { + return nil, err + } + + return res.Body(), nil +} diff --git a/pkg/shared/poetry/client.go b/pkg/microservice/picket/client/aslan/client.go similarity index 68% rename from pkg/shared/poetry/client.go rename to pkg/microservice/picket/client/aslan/client.go index 323db52acca20ac858ccd0856038e390009d26ea..42302493d54c6f9f8cd231965aed1f716b1f82aa 100644 --- a/pkg/shared/poetry/client.go +++ b/pkg/microservice/picket/client/aslan/client.go @@ -14,36 +14,28 @@ See the License for the specific language governing permissions and limitations under the License. */ -package poetry +package aslan import ( - "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/config" "github.com/koderover/zadig/pkg/tool/httpclient" ) -const ( - GitLabProvider = "gitlab" - GitHubProvider = "github" - GerritProvider = "gerrit" -) - type Client struct { *httpclient.Client - host string - token string + host string } -func New(host, token string) *Client { +func New() *Client { + host := config.AslanServiceAddress() + c := httpclient.New( - httpclient.SetAuthScheme(setting.RootAPIKey), - httpclient.SetAuthToken(token), - httpclient.SetHostURL(host), + httpclient.SetHostURL(host + "/api"), ) return &Client{ Client: c, host: host, - token: token, } } diff --git a/pkg/microservice/picket/client/aslan/delivery.go b/pkg/microservice/picket/client/aslan/delivery.go new file mode 100644 index 0000000000000000000000000000000000000000..5649c3ed1b0d2deb02606be5ca7a9ad104768191 --- /dev/null +++ b/pkg/microservice/picket/client/aslan/delivery.go @@ -0,0 +1,40 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aslan + +import ( + "net/http" + "net/url" + + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +func (c *Client) ListDelivery(header http.Header, qs url.Values, productName, workflowName, taskId, perPage, page string) ([]byte, error) { + url := "/delivery/releases" + queryParams := make(map[string]string) + queryParams["productName"] = productName + queryParams["workflowName"] = workflowName + queryParams["taskId"] = taskId + queryParams["per_page"] = perPage + queryParams["page"] = page + res, err := c.Get(url, httpclient.SetHeadersFromHTTPHeader(header), httpclient.SetQueryParamsFromValues(qs), httpclient.SetQueryParams(queryParams)) + if err != nil { + return nil, err + } + + return res.Body(), nil +} diff --git a/pkg/microservice/picket/client/aslan/kubeconfig.go b/pkg/microservice/picket/client/aslan/kubeconfig.go new file mode 100644 index 0000000000000000000000000000000000000000..6373970ad925efb5668c98fa514a204db7595adc --- /dev/null +++ b/pkg/microservice/picket/client/aslan/kubeconfig.go @@ -0,0 +1,35 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aslan + +import ( + "net/http" + "net/url" + + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +func (c *Client) DownloadKubeConfig(header http.Header, qs url.Values) ([]byte, error) { + url := "/setting/user/kube/config" + + res, err := c.Get(url, httpclient.SetHeadersFromHTTPHeader(header), httpclient.SetQueryParamsFromValues(qs)) + if err != nil { + return nil, err + } + + return res.Body(), nil +} diff --git a/pkg/microservice/picket/client/aslan/project.go b/pkg/microservice/picket/client/aslan/project.go new file mode 100644 index 0000000000000000000000000000000000000000..6751e8b1b7c804e514ba41fff33758060600633c --- /dev/null +++ b/pkg/microservice/picket/client/aslan/project.go @@ -0,0 +1,67 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aslan + +import ( + "fmt" + "net/http" + "net/url" + + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +func (c *Client) DeleteProject(header http.Header, qs url.Values, productName string) ([]byte, error) { + url := fmt.Sprintf("/project/products/%s", productName) + + res, err := c.Delete(url, httpclient.SetHeadersFromHTTPHeader(header), httpclient.SetQueryParamsFromValues(qs)) + if err != nil { + return nil, err + } + + return res.Body(), nil +} + +func (c *Client) ListProjects(header http.Header, qs url.Values) ([]byte, error) { + url := "/project/projects" + + res, err := c.Get(url, httpclient.SetHeadersFromHTTPHeader(header), httpclient.SetQueryParamsFromValues(qs)) + if err != nil { + return nil, err + } + + return res.Body(), nil +} + +func (c *Client) CreateProject(header http.Header, qs url.Values, body []byte) ([]byte, error) { + url := "/project/products" + res, err := c.Post(url, httpclient.SetHeadersFromHTTPHeader(header), httpclient.SetQueryParamsFromValues(qs), httpclient.SetBody(body)) + if err != nil { + return nil, err + } + + return res.Body(), nil +} + +func (c *Client) UpdateProject(header http.Header, qs url.Values, body []byte) ([]byte, error) { + url := "/project/products" + res, err := c.Put(url, httpclient.SetHeadersFromHTTPHeader(header), httpclient.SetQueryParamsFromValues(qs), httpclient.SetBody(body)) + if err != nil { + return nil, err + } + + return res.Body(), nil +} diff --git a/pkg/microservice/picket/client/aslan/stat.go b/pkg/microservice/picket/client/aslan/stat.go new file mode 100644 index 0000000000000000000000000000000000000000..8e90c2cdf6eb54a79cd2fcf7f3215a6c9afadcfe --- /dev/null +++ b/pkg/microservice/picket/client/aslan/stat.go @@ -0,0 +1,61 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aslan + +import ( + "net/http" + "net/url" + + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +func (c *Client) Overview(header http.Header, qs url.Values) ([]byte, error) { + url := "/stat/dashboard/overview" + res, err := c.Get(url, httpclient.SetHeadersFromHTTPHeader(header), httpclient.SetQueryParamsFromValues(qs)) + if err != nil { + return nil, err + } + + return res.Body(), nil +} +func (c *Client) Test(header http.Header, qs url.Values) ([]byte, error) { + url := "/stat/dashboard/test" + res, err := c.Get(url, httpclient.SetHeadersFromHTTPHeader(header), httpclient.SetQueryParamsFromValues(qs)) + if err != nil { + return nil, err + } + + return res.Body(), nil +} +func (c *Client) Deploy(header http.Header, qs url.Values) ([]byte, error) { + url := "/stat/dashboard/deploy" + res, err := c.Get(url, httpclient.SetHeadersFromHTTPHeader(header), httpclient.SetQueryParamsFromValues(qs)) + if err != nil { + return nil, err + } + + return res.Body(), nil +} +func (c *Client) Build(header http.Header, qs url.Values) ([]byte, error) { + url := "/stat/dashboard/build" + res, err := c.Get(url, httpclient.SetHeadersFromHTTPHeader(header), httpclient.SetQueryParamsFromValues(qs)) + if err != nil { + return nil, err + } + + return res.Body(), nil +} diff --git a/pkg/microservice/picket/client/aslan/workflow.go b/pkg/microservice/picket/client/aslan/workflow.go new file mode 100644 index 0000000000000000000000000000000000000000..7ae400f5d5cbb4c4705423a68756b626b9fec084 --- /dev/null +++ b/pkg/microservice/picket/client/aslan/workflow.go @@ -0,0 +1,116 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aslan + +import ( + "fmt" + "net/http" + "net/url" + "path" + + "github.com/koderover/zadig/pkg/tool/httpclient" + "github.com/koderover/zadig/pkg/tool/log" +) + +func (c *Client) ListWorkflows(header http.Header, qs url.Values) ([]byte, error) { + url := "/workflow/workflow" + + res, err := c.Get(url, httpclient.SetHeadersFromHTTPHeader(header), httpclient.SetQueryParamsFromValues(qs)) + if err != nil { + return nil, err + } + + return res.Body(), nil +} + +func (c *Client) ListWorkflowsV3(header http.Header, qs url.Values) ([]byte, error) { + url := "/workflow/v3" + + res, err := c.Get(url, httpclient.SetHeadersFromHTTPHeader(header), httpclient.SetQueryParamsFromValues(qs)) + if err != nil { + return nil, err + } + + return res.Body(), nil +} + +func (c *Client) ListTestWorkflows(testName string, header http.Header, qs url.Values) ([]byte, error) { + url := fmt.Sprintf("/workflow/workflow/testName/%s", testName) + + res, err := c.Get(url, httpclient.SetHeadersFromHTTPHeader(header), httpclient.SetQueryParamsFromValues(qs)) + if err != nil { + return nil, err + } + + return res.Body(), nil +} + +func (c *Client) CreateWorkflowTask(header http.Header, qs url.Values, body []byte, workflowName string) ([]byte, error) { + url := path.Join("/workflow/workflowtask", workflowName) + + res, err := c.Post(url, httpclient.SetHeadersFromHTTPHeader(header), httpclient.SetQueryParamsFromValues(qs), httpclient.SetBody(body)) + if err != nil { + return nil, err + } + + return res.Body(), nil +} + +func (c *Client) CancelWorkflowTask(header http.Header, qs url.Values, id string, name string) (statusCode int, err error) { + url := fmt.Sprintf("/workflow/workflowtask/id/%s/pipelines/%s", id, name) + + res, err := c.Delete(url, httpclient.SetHeadersFromHTTPHeader(header), httpclient.SetQueryParamsFromValues(qs)) + if err != nil { + return http.StatusInternalServerError, err + } + + return res.StatusCode(), nil +} + +func (c *Client) RestartWorkflowTask(header http.Header, qs url.Values, id string, name string) (statusCode int, err error) { + url := fmt.Sprintf("/workflow/workflowtask/id/%s/pipelines/%s/restart", id, name) + + res, err := c.Post(url, httpclient.SetHeadersFromHTTPHeader(header), httpclient.SetQueryParamsFromValues(qs)) + if err != nil { + log.Errorf("RestartWorkflowTask err: %s,res: %s", err, string(res.Body())) + return http.StatusInternalServerError, err + } + + return res.StatusCode(), nil +} + +func (c *Client) ListWorkflowTask(header http.Header, qs url.Values, commitId string) ([]byte, error) { + url := fmt.Sprintf("/workflow/v2/status/task/info?commitId=%s", commitId) + + res, err := c.Get(url, httpclient.SetHeadersFromHTTPHeader(header), httpclient.SetQueryParamsFromValues(qs)) + if err != nil { + return nil, err + } + + return res.Body(), nil +} + +func (c *Client) GetDetailedWorkflowTask(header http.Header, qs url.Values, taskID, name string) ([]byte, error) { + url := fmt.Sprintf("/workflow/workflowtask/id/%s/pipelines/%s", taskID, name) + + res, err := c.Get(url, httpclient.SetHeadersFromHTTPHeader(header), httpclient.SetQueryParamsFromValues(qs)) + if err != nil { + return nil, err + } + + return res.Body(), nil +} diff --git a/pkg/microservice/picket/client/opa/evaluate.go b/pkg/microservice/picket/client/opa/evaluate.go index f5920ae629c829680dfbc99d6bef1e78d76a2d59..7d4ae908d367bbad43d23cdcb5b987c9c775776b 100644 --- a/pkg/microservice/picket/client/opa/evaluate.go +++ b/pkg/microservice/picket/client/opa/evaluate.go @@ -1,3 +1,19 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package opa import ( @@ -7,20 +23,27 @@ import ( "github.com/koderover/zadig/pkg/tool/httpclient" ) +type InputGenerator func() (*Input, error) + // Evaluate evaluates the query with the given input and return a json response which has a field called "result" -func (c *Client) Evaluate(query string, input interface{}) ([]byte, error) { +func (c *Client) Evaluate(query string, result interface{}, ig InputGenerator) error { + input, err := ig() + if err != nil { + return err + } + url := "v1/data" - req := struct{ + req := struct { Input interface{} `json:"input"` }{ Input: input, } queryURL := fmt.Sprintf("%s/%s", url, strings.ReplaceAll(query, ".", "/")) - res, err := c.Post(queryURL, httpclient.SetBody(req)) + _, err = c.Post(queryURL, httpclient.SetBody(req), httpclient.SetResult(result)) if err != nil { - return nil, err + return err } - return res.Body(), nil + return nil } diff --git a/pkg/microservice/picket/client/opa/input.go b/pkg/microservice/picket/client/opa/input.go new file mode 100644 index 0000000000000000000000000000000000000000..fafb4a2f98d3439e1b6aa1840a2964314b8a0a78 --- /dev/null +++ b/pkg/microservice/picket/client/opa/input.go @@ -0,0 +1,39 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package opa + +type Input struct { + ParsedQuery *ParseQuery `json:"parsed_query"` + ParsedPath []string `json:"parsed_path"` + Attributes *Attributes `json:"attributes"` +} +type ParseQuery struct { + ProjectName []string `json:"projectName"` +} + +type Attributes struct { + Request *Request `json:"request"` +} + +type Request struct { + HTTP *HTTPSpec `json:"http"` +} + +type HTTPSpec struct { + Method string `json:"method"` + Headers map[string]string `json:"headers"` +} diff --git a/pkg/tool/ilyshin/client.go b/pkg/microservice/picket/client/policy/client.go similarity index 65% rename from pkg/tool/ilyshin/client.go rename to pkg/microservice/picket/client/policy/client.go index 03a9d3a4162d7dd77f1189d487a2cfd36e4ecccf..9790686fa1f923a8db40d2625f948acc8aeab5ac 100644 --- a/pkg/tool/ilyshin/client.go +++ b/pkg/microservice/picket/client/policy/client.go @@ -14,30 +14,28 @@ See the License for the specific language governing permissions and limitations under the License. */ -package ilyshin +package policy import ( - "crypto/tls" - + "github.com/koderover/zadig/pkg/config" "github.com/koderover/zadig/pkg/tool/httpclient" ) type Client struct { *httpclient.Client - Address string - AccessToken string + + host string } -func NewClient(address, accessToken string) *Client { +func New() *Client { + host := config.PolicyServiceAddress() + c := httpclient.New( - httpclient.SetAuthScheme("Bearer"), - httpclient.SetAuthToken(accessToken), - httpclient.SetHostURL(address), - httpclient.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}), + httpclient.SetHostURL(host + "/api/v1"), ) + return &Client{ - Client: c, - Address: address, - AccessToken: accessToken, + Client: c, + host: host, } } diff --git a/pkg/microservice/picket/client/policy/rolebinding.go b/pkg/microservice/picket/client/policy/rolebinding.go new file mode 100644 index 0000000000000000000000000000000000000000..bb8bbe39b44f9c175695bf00c2bab796391d34d8 --- /dev/null +++ b/pkg/microservice/picket/client/policy/rolebinding.go @@ -0,0 +1,94 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package policy + +import ( + "net/http" + "net/url" + + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/httpclient" + "github.com/koderover/zadig/pkg/types" +) + +type RoleBinding struct { + Name string `json:"name"` + UID string `json:"uid"` + Role string `json:"role"` + Preset bool `json:"preset"` + Type setting.ResourceType `json:"type"` +} + +type PolicyBinding struct { + Name string `json:"name"` + UID string `json:"uid"` + Policy string `json:"policy"` + Preset bool `json:"preset"` + Type setting.ResourceType `json:"type"` +} + +func (c *Client) ListRoleBindings(header http.Header, qs url.Values) ([]*RoleBinding, error) { + url := "/rolebindings" + + res := make([]*RoleBinding, 0) + _, err := c.Get(url, httpclient.SetHeadersFromHTTPHeader(header), httpclient.SetQueryParamsFromValues(qs), httpclient.SetResult(&res)) + if err != nil { + return nil, err + } + + return res, nil +} + +func (c *Client) ListPolicyBindings(header http.Header, qs url.Values) ([]*PolicyBinding, error) { + url := "/policybindings" + + res := make([]*PolicyBinding, 0) + _, err := c.Get(url, httpclient.SetHeadersFromHTTPHeader(header), httpclient.SetQueryParamsFromValues(qs), httpclient.SetResult(&res)) + if err != nil { + return nil, err + } + + return res, nil +} + +func (c *Client) DeleteRoleBindings(userID string, header http.Header, qs url.Values) ([]byte, error) { + url := "rolebindings/bulk-delete" + + qs.Add("userID", userID) + res, err := c.Post(url, httpclient.SetHeadersFromHTTPHeader(header), httpclient.SetQueryParamsFromValues(qs), httpclient.SetBody([]byte("{}"))) + if err != nil { + return []byte{}, err + } + return res.Body(), err +} + +type SearchSystemRoleBindingArgs struct { + Uids []string `json:"uids"` +} + +func (c *Client) SearchSystemRoleBindings(uids []string) (map[string][]*types.RoleBinding, error) { + url := "system-rolebindings/search" + args := SearchSystemRoleBindingArgs{ + Uids: uids, + } + result := map[string][]*types.RoleBinding{} + _, err := c.Post(url, httpclient.SetBody(args), httpclient.SetResult(&result)) + if err != nil { + return nil, err + } + return result, nil +} diff --git a/pkg/microservice/picket/client/user/client.go b/pkg/microservice/picket/client/user/client.go new file mode 100644 index 0000000000000000000000000000000000000000..5e6d8ac15e89bad3c8d9917c9d714db7b45001be --- /dev/null +++ b/pkg/microservice/picket/client/user/client.go @@ -0,0 +1,41 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package user + +import ( + "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +type Client struct { + *httpclient.Client + + host string +} + +func New() *Client { + host := config.UserServiceAddress() + + c := httpclient.New( + httpclient.SetHostURL(host + "/api/v1"), + ) + + return &Client{ + Client: c, + host: host, + } +} diff --git a/pkg/microservice/picket/client/user/user.go b/pkg/microservice/picket/client/user/user.go new file mode 100644 index 0000000000000000000000000000000000000000..7a5d623f70dd4930b50487f874bbe113bb74a0ce --- /dev/null +++ b/pkg/microservice/picket/client/user/user.go @@ -0,0 +1,58 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package user + +import ( + "net/http" + "net/url" + + "github.com/koderover/zadig/pkg/tool/httpclient" + "github.com/koderover/zadig/pkg/types" +) + +func (c *Client) DeleteUser(userId string, header http.Header, qs url.Values) ([]byte, error) { + url := "/users/" + userId + + res, err := c.Delete(url, httpclient.SetHeadersFromHTTPHeader(header), httpclient.SetQueryParamsFromValues(qs)) + if err != nil { + return nil, err + } + + return res.Body(), nil +} + +func (c *Client) SearchUsers(header http.Header, qs url.Values, body interface{}) (*types.UsersResp, error) { + url := "/users/search" + result := &types.UsersResp{} + _, err := c.Post(url, httpclient.SetHeadersFromHTTPHeader(header), httpclient.SetQueryParamsFromValues(qs), httpclient.SetBody(body), httpclient.SetResult(result)) + if err != nil { + return nil, err + } + return result, nil +} + +type SearchArgs struct { + Name string `json:"name,omitempty"` + Account string `json:"account,omitempty"` + IdentityType string `json:"identity_type,omitempty"` + UIDs []string `json:"uids,omitempty"` + PerPage int `json:"per_page,omitempty"` + Page int `json:"page,omitempty"` +} + +type SearchRes struct { +} diff --git a/pkg/microservice/picket/config/consts.go b/pkg/microservice/picket/config/consts.go new file mode 100644 index 0000000000000000000000000000000000000000..9441030192842fcc7ef09716bd5f1661ffdc5dad --- /dev/null +++ b/pkg/microservice/picket/config/consts.go @@ -0,0 +1,24 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +type RulesLogicalOperator string + +const ( + AND RulesLogicalOperator = "and" + OR RulesLogicalOperator = "or" +) diff --git a/pkg/microservice/picket/core/evaluation/service/evaluate.go b/pkg/microservice/picket/core/evaluation/service/evaluate.go index 4de488cb897f076a535e9313b69d66a494139beb..867e059e682b60cf40eb7a6660224b0e6a7e9783 100644 --- a/pkg/microservice/picket/core/evaluation/service/evaluate.go +++ b/pkg/microservice/picket/core/evaluation/service/evaluate.go @@ -17,66 +17,36 @@ limitations under the License. package service import ( - "encoding/json" "net/http" "strings" "go.uber.org/zap" "github.com/koderover/zadig/pkg/microservice/picket/client/opa" + "github.com/koderover/zadig/pkg/setting" ) -type input struct { - ParsedQuery parseQuery `json:"parsed_query"` - ParsedPath []string `json:"parsed_path"` - Attributes attributes `json:"attributes"` -} -type parseQuery struct { - ProjectName []string `json:"projectName"` -} - -type attributes struct { - Request request `json:"request"` -} - -type request struct { - Http HTTP `json:"http"` -} - -type HTTP struct { - Method string `json:"method"` - Headers map[string]string `json:"headers"` -} - -type opaRes struct { +type evaluateResult struct { Result bool `json:"result"` } func Evaluate(header http.Header, projectName string, grantsReq []GrantReq, logger *zap.SugaredLogger) ([]GrantRes, error) { - // 拼接参数，请求opa - authorization := header.Get("authorization") - opaHeaders := map[string]string{} - opaHeaders["authorization"] = authorization var grantsRes []GrantRes opaClient := opa.NewDefault() // 对于每一个action+endpoint 都去请求opa for _, v := range grantsReq { parsedPath := strings.Split(strings.Trim(v.EndPoint, "/"), "/") - input := generateOPAInput(projectName, parsedPath, v.Method, opaHeaders) - res, err := opaClient.Evaluate("rbac.allow", input) + res := &evaluateResult{} + err := opaClient.Evaluate( + "rbac.allow", res, + func() (*opa.Input, error) { return generateOPAInput(header, projectName, v.Method, parsedPath), nil }) if err != nil { logger.Errorf("opa evaluate endpoint: %v method: %v err: %s", v.EndPoint, v.Method, err) grantsRes = append(grantsRes, GrantRes{v, false}) continue } - var opaR opaRes - if err := json.Unmarshal(res, &opaR); err != nil { - logger.Errorf("opa res Unmarshal err %s", err) - grantsRes = append(grantsRes, GrantRes{v, false}) - continue - } - grantsRes = append(grantsRes, GrantRes{v, opaR.Result}) + grantsRes = append(grantsRes, GrantRes{v, res.Result}) } return grantsRes, nil } @@ -91,16 +61,20 @@ type GrantRes struct { Allow bool `json:"allow"` } -func generateOPAInput(projectName string, parsedPath []string, method string, head map[string]string) input { - return input{ - ParsedQuery: parseQuery{ +func generateOPAInput(header http.Header, projectName, method string, parsedPath []string) *opa.Input { + authorization := header.Get(strings.ToLower(setting.AuthorizationHeader)) + headers := map[string]string{} + headers[strings.ToLower(setting.AuthorizationHeader)] = authorization + + return &opa.Input{ + ParsedQuery: &opa.ParseQuery{ ProjectName: []string{projectName}, }, ParsedPath: parsedPath, - Attributes: attributes{ - Request: request{Http: HTTP{ + Attributes: &opa.Attributes{ + Request: &opa.Request{HTTP: &opa.HTTPSpec{ Method: method, - Headers: head, + Headers: headers, }}, }, } diff --git a/pkg/microservice/picket/core/filter/handler/kubeconfig.go b/pkg/microservice/picket/core/filter/handler/kubeconfig.go new file mode 100644 index 0000000000000000000000000000000000000000..365e75e5f2e3ab18ce96b70b10d44670ce30386a --- /dev/null +++ b/pkg/microservice/picket/core/filter/handler/kubeconfig.go @@ -0,0 +1,31 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/picket/core/filter/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" +) + +func GetKubeConfig(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.GetKubeConfig(c.Request.Header, c.Request.URL.Query(), ctx.Logger) +} diff --git a/pkg/microservice/picket/core/filter/handler/project.go b/pkg/microservice/picket/core/filter/handler/project.go new file mode 100644 index 0000000000000000000000000000000000000000..a0c556525027e9037ee10fe8d1316cf29ed14559 --- /dev/null +++ b/pkg/microservice/picket/core/filter/handler/project.go @@ -0,0 +1,81 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + "github.com/gin-gonic/gin/binding" + + "github.com/koderover/zadig/pkg/microservice/picket/core/filter/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +func ListProjects(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.ListProjects(c.Request.Header, c.Request.URL.Query(), ctx.Logger) +} + +func CreateProject(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := &service.CreateProjectArgs{} + if err := c.ShouldBindBodyWith(args, binding.JSON); err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err).AddDesc("invalid CreateProjectReq") + return + } + body := getReqBody(c) + + ctx.Resp, ctx.Err = service.CreateProject(c.Request.Header, body, c.Request.URL.Query(), args, ctx.Logger) +} + +func getReqBody(c *gin.Context) (body []byte) { + if cb, ok := c.Get(gin.BodyBytesKey); ok { + if cbb, ok := cb.([]byte); ok { + body = cbb + } + } + return body +} + +type UpdateProjectReq struct { + Public bool `json:"public"` + ProductName string `json:"product_name"` +} + +func UpdateProject(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(UpdateProjectReq) + if err := c.ShouldBindBodyWith(args, binding.JSON); err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err).AddDesc("invalid UpdateProject") + return + } + body := getReqBody(c) + ctx.Resp, ctx.Err = service.UpdateProject(c.Request.Header, c.Request.URL.Query(), body, args.ProductName, args.Public, ctx.Logger) +} + +func DeleteProject(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.DeleteProject(c.Request.Header, c.Request.URL.Query(), c.Param("name"), ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/common/handler/token.go b/pkg/microservice/picket/core/filter/handler/rolebinding.go similarity index 54% rename from pkg/microservice/aslan/core/common/handler/token.go rename to pkg/microservice/picket/core/filter/handler/rolebinding.go index 7fee3260bcfccd4c8de0c7e981b1dab902d0c452..4a857af34703c91fb33eb155a25fb4d1466a7598 100644 --- a/pkg/microservice/aslan/core/common/handler/token.go +++ b/pkg/microservice/picket/core/filter/handler/rolebinding.go @@ -17,26 +17,22 @@ limitations under the License. package handler import ( - "net/http" - "strings" - "github.com/gin-gonic/gin" - "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/microservice/picket/core/filter/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" ) -func GetToken(c *gin.Context) { - authorization := c.Request.Header.Get(setting.AuthorizationHeader) - if authorization != "" { - if strings.HasPrefix(authorization, setting.UserAPIKey+" ") { - token := strings.Split(authorization, " ")[1] - c.JSON(http.StatusOK, gin.H{"token": token}) - return - } - } else if token, err := c.Cookie("TOKEN"); err == nil { - c.JSON(http.StatusOK, gin.H{"token": token}) +func ListBindings(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("projectName is empty") return } - c.JSON(http.StatusBadRequest, gin.H{"message": "illegal request"}) + ctx.Resp, ctx.Err = service.ListBindings(c.Request.Header, c.Request.URL.Query(), ctx.Logger) } diff --git a/pkg/microservice/picket/core/filter/handler/router.go b/pkg/microservice/picket/core/filter/handler/router.go new file mode 100644 index 0000000000000000000000000000000000000000..58274a7d95040342a0ddc3c1f95dc8376aa91580 --- /dev/null +++ b/pkg/microservice/picket/core/filter/handler/router.go @@ -0,0 +1,61 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" +) + +type Router struct{} + +func (*Router) Inject(router *gin.RouterGroup) { + projects := router.Group("projects") + { + projects.DELETE("/:name", DeleteProject) + projects.GET("", ListProjects) + projects.POST("", CreateProject) + projects.PUT("/:name", UpdateProject) + } + + workflows := router.Group("workflows") + { + workflows.GET("testName/:testName", ListTestWorkflows) + workflows.GET("", ListWorkflows) + workflows.GET("v3", ListWorkflowsV3) + } + + rolebindings := router.Group("bindings") + { + rolebindings.GET("", ListBindings) + } + users := router.Group("users") + { + users.POST("", SearchUsers) + users.DELETE("/:id", DeleteUser) + } + downloads := router.Group("kubeconfig") + { + downloads.GET("", GetKubeConfig) + } + stat := router.Group("stat") + { + stat.GET("dashboard/overview", Overview) + stat.GET("dashboard/deploy", Deploy) + stat.GET("dashboard/test", Test) + stat.GET("dashboard/build", Build) + } +} diff --git a/pkg/microservice/picket/core/filter/handler/stat.go b/pkg/microservice/picket/core/filter/handler/stat.go new file mode 100644 index 0000000000000000000000000000000000000000..a9cae8d1333a07d731ac1658e1b34e5e67680821 --- /dev/null +++ b/pkg/microservice/picket/core/filter/handler/stat.go @@ -0,0 +1,49 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/picket/core/filter/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" +) + +func Overview(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.Overview(c.Request.Header, c.Request.URL.Query(), ctx.Logger) +} +func Test(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.Test(c.Request.Header, c.Request.URL.Query(), ctx.Logger) +} +func Deploy(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.Deploy(c.Request.Header, c.Request.URL.Query(), ctx.Logger) +} +func Build(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.Build(c.Request.Header, c.Request.URL.Query(), ctx.Logger) +} diff --git a/pkg/microservice/picket/core/filter/handler/users.go b/pkg/microservice/picket/core/filter/handler/users.go new file mode 100644 index 0000000000000000000000000000000000000000..f6f85dc5af9094b4ab1e03332c9edcb16857e7d5 --- /dev/null +++ b/pkg/microservice/picket/core/filter/handler/users.go @@ -0,0 +1,47 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/picket/client/user" + "github.com/koderover/zadig/pkg/microservice/picket/core/filter/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" +) + +func DeleteUser(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + uid := c.Param("id") + ctx.Resp, ctx.Err = service.DeleteUser(uid, c.Request.Header, c.Request.URL.Query(), ctx.Logger) +} + +func SearchUsers(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + req := &user.SearchArgs{} + + if err := c.ShouldBindJSON(req); err != nil { + ctx.Logger.Errorf("bindjson err :%s", err) + ctx.Err = err + return + } + ctx.Resp, ctx.Err = service.SearchUsers(c.Request.Header, c.Request.URL.Query(), req, ctx.Logger) +} diff --git a/pkg/microservice/picket/core/filter/handler/workflow.go b/pkg/microservice/picket/core/filter/handler/workflow.go new file mode 100644 index 0000000000000000000000000000000000000000..c405c8693479b7c244c3ec94f8f6d5d56eb63abe --- /dev/null +++ b/pkg/microservice/picket/core/filter/handler/workflow.go @@ -0,0 +1,46 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/picket/core/filter/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" +) + +func ListTestWorkflows(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + testName := c.Param("testName") + ctx.Resp, ctx.Err = service.ListTestWorkflows(testName, c.Request.Header, c.Request.URL.Query(), ctx.Logger) +} + +func ListWorkflows(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.ListWorkflows(c.Request.Header, c.Request.URL.Query(), ctx.Logger) +} + +func ListWorkflowsV3(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.ListWorkflowsV3(c.Request.Header, c.Request.URL.Query(), ctx.Logger) +} diff --git a/pkg/tool/ilyshin/commit.go b/pkg/microservice/picket/core/filter/service/kubeconfig.go similarity index 31% rename from pkg/tool/ilyshin/commit.go rename to pkg/microservice/picket/core/filter/service/kubeconfig.go index e90f7a76c441564ccb94c4839b92f6e378bbf762..b020f970761fc518005fa0e089e8a39b0aa396e8 100644 --- a/pkg/tool/ilyshin/commit.go +++ b/pkg/microservice/picket/core/filter/service/kubeconfig.go @@ -14,48 +14,52 @@ See the License for the specific language governing permissions and limitations under the License. */ -package ilyshin +package service import ( - "fmt" - "time" + "net/http" + "net/url" "go.uber.org/zap" - "github.com/koderover/zadig/pkg/tool/httpclient" + "github.com/koderover/zadig/pkg/microservice/picket/client/aslan" + "github.com/koderover/zadig/pkg/microservice/picket/config" ) -type Commit struct { - ID string `json:"id"` - ShortID string `json:"short_id"` - Title string `json:"title"` - Message string `json:"message"` - AuthorName string `json:"author_name"` - AuthorEmail string `json:"author_email"` - AuthoredDate *time.Time `json:"authored_date"` - CommitterName string `json:"committer_name"` - CommitterEmail string `json:"committer_email"` - CommittedDate *time.Time `json:"committed_date"` - CreatedAt *time.Time `json:"created_at"` -} +//DownloadKubeConfig user download kube config file which has permission to read or edit namespaces he has permission to +//query the opa service to get the project lists by pass through *rules parameter action +func GetKubeConfig(header http.Header, qs url.Values, logger *zap.SugaredLogger) ([]byte, error) { + readEnvRules := []*rule{{ + method: "GET", + endpoint: "/api/aslan/environment/environments", + }} + editEnvRules := []*rule{{ + method: "PUT", + endpoint: "/api/aslan/environment/environments/?*", + }, { + method: "POST", + endpoint: "/api/aslan/environment/environments/?*/services/?*/restart", + }, + } + projectsEnvCanView, err := getAllowedProjects(header, readEnvRules, config.OR, logger) + if err != nil { + logger.Errorf("Failed to get allowed project names, err: %s", err) + return nil, err + } -func (c *Client) GetLatestCommit(owner, repo, branch, path string, log *zap.SugaredLogger) (*Commit, error) { - url := fmt.Sprintf("/api/v4/projects/%s/repository/commits", generateProjectName(owner, repo)) - qs := map[string]string{ - "per_page": "1", - "ref_name": branch, - "path": path, + for _, name := range projectsEnvCanView { + qs.Add("projectsEnvCanView", name) } - var err error - var commits []*Commit - if _, err = c.Get(url, httpclient.SetQueryParams(qs), httpclient.SetResult(&commits)); err != nil { - log.Errorf("Failed to list project commits, error: %s", err) + projectsEnvCanEdit, err := getAllowedProjects(header, editEnvRules, config.OR, logger) + if err != nil { + logger.Errorf("Failed to get allowed project names, err: %s", err) return nil, err } - if len(commits) == 0 { - return nil, fmt.Errorf("not found commits") + + for _, name := range projectsEnvCanEdit { + qs.Add("projectsEnvCanEdit", name) } - return commits[0], nil + return aslan.New().DownloadKubeConfig(header, qs) } diff --git a/pkg/microservice/picket/core/filter/service/project.go b/pkg/microservice/picket/core/filter/service/project.go new file mode 100644 index 0000000000000000000000000000000000000000..e62a0f9671a258b4973d4a00382956d624c08bb3 --- /dev/null +++ b/pkg/microservice/picket/core/filter/service/project.go @@ -0,0 +1,176 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "net/http" + "net/url" + "strings" + + "go.uber.org/zap" + + configbase "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/picket/client/aslan" + "github.com/koderover/zadig/pkg/microservice/picket/client/opa" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/client/policy" + "github.com/koderover/zadig/pkg/tool/log" +) + +type CreateProjectArgs struct { + Public bool `json:"public"` + ProductName string `json:"product_name"` + Admins []string `json:"admins"` +} + +type allowedProjectsData struct { + Result []string `json:"result"` +} + +func CreateProject(header http.Header, body []byte, qs url.Values, args *CreateProjectArgs, logger *zap.SugaredLogger) ([]byte, error) { + var rbs []*policy.RoleBinding + for _, uid := range args.Admins { + rbs = append(rbs, &policy.RoleBinding{ + Name: configbase.RoleBindingNameFromUIDAndRole(uid, setting.ProjectAdmin, ""), + UID: uid, + Role: string(setting.ProjectAdmin), + Preset: true, + }) + } + + if args.Public { + rbs = append(rbs, &policy.RoleBinding{ + Name: configbase.RoleBindingNameFromUIDAndRole("*", setting.ReadOnly, ""), + UID: "*", + Role: string(setting.ReadOnly), + Preset: true, + }) + } + + policyClient := policy.NewDefault() + for _, rb := range rbs { + if err := policyClient.CreateOrUpdateRoleBinding(args.ProductName, rb); err != nil { + logger.Errorf("Failed to create rolebinding %s, err: %s", rb.Name, err) + return nil, err + } + } + + res, err := aslan.New().CreateProject(header, qs, body) + if err != nil { + logger.Errorf("Failed to create project %s, err: %s", args.ProductName, err) + for _, rb := range rbs { + if err1 := policyClient.DeleteRoleBinding(rb.Name, args.ProductName); err1 != nil { + logger.Errorf("Failed to create rolebinding %s, err: %s", rb.Name, err1) + } + } + + return nil, err + } + + return res, nil +} + +func UpdateProject(header http.Header, qs url.Values, body []byte, projectName string, public bool, logger *zap.SugaredLogger) ([]byte, error) { + // role binding + roleBindingName := configbase.RoleBindingNameFromUIDAndRole("*", setting.ReadOnly, "") + if !public { + if err := policy.NewDefault().DeleteRoleBinding(roleBindingName, projectName); err != nil { + logger.Errorf("Failed to delete role binding, err: %s", err) + return nil, err + } + } else { + if err := policy.NewDefault().CreateOrUpdateRoleBinding(projectName, &policy.RoleBinding{ + Name: roleBindingName, + UID: "*", + Role: string(setting.ReadOnly), + Preset: true, + }); err != nil { + logger.Errorf("Failed to create role binding, err: %s", err) + return nil, err + } + } + + res, err := aslan.New().UpdateProject(header, qs, body) + if err != nil { + logger.Errorf("Failed to create project %s, err: %s", projectName, err) + return nil, err + } + + return res, nil +} + +func ListProjects(header http.Header, qs url.Values, logger *zap.SugaredLogger) ([]byte, error) { + names, err := getVisibleProjects(header, logger) + if err != nil { + logger.Errorf("Failed to get allowed project names, err: %s", err) + return nil, err + } + + if len(names) == 0 { + return []byte("[]"), nil + } + + if !(len(names) == 1 && names[0] == "*") { + for _, name := range names { + qs.Add("names", name) + } + } + + aslanClient := aslan.New() + + return aslanClient.ListProjects(header, qs) +} + +func DeleteProject(header http.Header, qs url.Values, productName string, logger *zap.SugaredLogger) ([]byte, error) { + // delete roles and rolebindings + if err := policy.NewDefault().DeleteRoleBindings([]string{"*"}, productName); err != nil { + log.Errorf("delete rolebindings err:%s", err) + } + if err := policy.NewDefault().DeleteRoles([]string{"*"}, productName); err != nil { + log.Errorf("delete roles err:%s", err) + } + return aslan.New().DeleteProject(header, qs, productName) +} + +func getVisibleProjects(headers http.Header, logger *zap.SugaredLogger) ([]string, error) { + res := &allowedProjectsData{} + opaClient := opa.NewDefault() + err := opaClient.Evaluate("rbac.user_visible_projects", res, func() (*opa.Input, error) { return generateOPAInput(headers, "", ""), nil }) + if err != nil { + logger.Errorf("opa evaluation failed, err: %s", err) + return nil, err + } + + return res.Result, nil +} + +func generateOPAInput(header http.Header, method string, endpoint string) *opa.Input { + authorization := header.Get(strings.ToLower(setting.AuthorizationHeader)) + headers := map[string]string{} + parsedPath := strings.Split(strings.Trim(endpoint, "/"), "/") + headers[strings.ToLower(setting.AuthorizationHeader)] = authorization + + return &opa.Input{ + Attributes: &opa.Attributes{ + Request: &opa.Request{HTTP: &opa.HTTPSpec{ + Headers: headers, + Method: method, + }}, + }, + ParsedPath: parsedPath, + } +} diff --git a/pkg/microservice/picket/core/filter/service/rolebinding.go b/pkg/microservice/picket/core/filter/service/rolebinding.go new file mode 100644 index 0000000000000000000000000000000000000000..7fc4385377d87ce35824295a1ed4c2da51c6c29c --- /dev/null +++ b/pkg/microservice/picket/core/filter/service/rolebinding.go @@ -0,0 +1,165 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "net/http" + "net/url" + + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/koderover/zadig/pkg/microservice/picket/client/policy" + "github.com/koderover/zadig/pkg/shared/client/user" +) + +const ALLUsers = "*" + +type roleBinding struct { + *policy.RoleBinding + Username string `json:"username"` + Email string `json:"email"` + Phone string `json:"phone"` + IdentityType string `json:"identity_type"` + Account string `json:"account"` +} + +type policyBinding struct { + *policy.PolicyBinding + Username string `json:"username"` + Email string `json:"email"` + Phone string `json:"phone"` + IdentityType string `json:"identity_type"` + Account string `json:"account"` +} + +type Binding struct { + Roles []*roleBinding `json:"roles"` + Policies []*policyBinding `json:"policies"` + UserName string `json:"username"` + Email string `json:"email"` + Account string `json:"account"` + IdentityType string `json:"identity_type"` + Phone string `json:"phone"` + Uid string `json:"uid"` +} + +func ListBindings(header http.Header, qs url.Values, logger *zap.SugaredLogger) ([]*Binding, error) { + rbs, err := policy.New().ListRoleBindings(header, qs) + if err != nil { + logger.Errorf("Failed to list rolebindings, err: %s", err) + return nil, err + } + + uidSets := sets.String{} + uidToRoleBinding := make(map[string][]*roleBinding) + for _, rb := range rbs { + if rb.UID != ALLUsers { + uidSets.Insert(rb.UID) + } + uidToRoleBinding[rb.UID] = append(uidToRoleBinding[rb.UID], &roleBinding{RoleBinding: rb}) + } + + pbs, err := policy.New().ListPolicyBindings(header, qs) + if err != nil { + logger.Errorf("Failed to list policybindings, err: %s", err) + return nil, err + } + + uidToPolicyBindings := make(map[string][]*policyBinding) + for _, pb := range pbs { + if pb.UID != ALLUsers { + uidSets.Insert(pb.UID) + } + uidToPolicyBindings[pb.UID] = append(uidToPolicyBindings[pb.UID], &policyBinding{PolicyBinding: pb}) + } + if uidSets.Len() == 0 { + if len(uidToRoleBinding[ALLUsers]) != 0 || len(uidToPolicyBindings[ALLUsers]) != 0 { + // add all 'ALLUsers' roles + AllUserBinding := &Binding{ + Roles: uidToRoleBinding[ALLUsers], + Policies: uidToPolicyBindings[ALLUsers], + UserName: "*", + Email: "", + Uid: "*", + } + return []*Binding{AllUserBinding}, nil + } + return []*Binding{}, nil + } + users, err := user.New().ListUsers(&user.SearchArgs{UIDs: uidSets.List()}) + if err != nil { + logger.Errorf("Failed to list users, err: %s", err) + return nil, err + } + + var res []*Binding + + for _, u := range users { + var policyBindings []*policyBinding + var roleBindings []*roleBinding + binding := &Binding{ + UserName: u.Name, + Email: u.Email, + IdentityType: u.IdentityType, + Account: u.Account, + Phone: u.Phone, + Uid: u.UID, + } + if rb, ok := uidToRoleBinding[u.UID]; ok { + for _, r := range rb { + r.Username = u.Name + r.Email = u.Email + r.Phone = u.Phone + r.IdentityType = u.IdentityType + r.Account = u.Account + roleBindings = append(roleBindings, r) + } + binding.Roles = roleBindings + } else { + logger.Warnf("can not find user's role bindings , uid:%v", u.UID) + } + if pb, ok := uidToPolicyBindings[u.UID]; ok { + for _, p := range pb { + p.Username = u.Name + p.Email = u.Email + p.Phone = u.Phone + p.IdentityType = u.IdentityType + p.Account = u.Account + policyBindings = append(policyBindings, p) + } + binding.Policies = policyBindings + } else { + logger.Warnf("can not find user's policy bindings , uid:%v", u.UID) + } + res = append(res, binding) + } + + // add all 'ALLUsers' roles + if len(uidToRoleBinding[ALLUsers]) != 0 || len(uidToPolicyBindings[ALLUsers]) != 0 { + AllUserBinding := &Binding{ + Roles: uidToRoleBinding[ALLUsers], + Policies: uidToPolicyBindings[ALLUsers], + UserName: "*", + Email: "", + Uid: "*", + } + res = append(res, AllUserBinding) + } + + return res, nil +} diff --git a/pkg/tool/ilyshin/group_project.go b/pkg/microservice/picket/core/filter/service/stat.go similarity index 45% rename from pkg/tool/ilyshin/group_project.go rename to pkg/microservice/picket/core/filter/service/stat.go index ffcb7ca354bdc2a1cdfef593018eb5e1b56d337f..bc709fc1c4dac67b16551e3a5e946a38267180da 100644 --- a/pkg/tool/ilyshin/group_project.go +++ b/pkg/microservice/picket/core/filter/service/stat.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The KodeRover Authors. +Copyright 2022 The KodeRover Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,32 +14,29 @@ See the License for the specific language governing permissions and limitations under the License. */ -package ilyshin +package service import ( - "fmt" + "net/http" + "net/url" "go.uber.org/zap" - "github.com/koderover/zadig/pkg/tool/httpclient" + "github.com/koderover/zadig/pkg/microservice/picket/client/aslan" ) -func (c *Client) ListGroupProjects(namespace, keyword string, log *zap.SugaredLogger) ([]*Project, error) { - url := fmt.Sprintf("/api/v4/groups/%s/projects", namespace) - qs := map[string]string{ - "order_by": "name", - "sort": "asc", - "per_page": "100", - } - if keyword != "" && len(keyword) > 2 { - qs["search"] = keyword - } - var err error - var gps []*Project - if _, err = c.Get(url, httpclient.SetQueryParams(qs), httpclient.SetResult(&gps)); err != nil { - log.Errorf("Failed to list group projects, error: %s", err) - return gps, err - } - - return gps, nil +func Overview(header http.Header, qs url.Values, log *zap.SugaredLogger) ([]byte, error) { + return aslan.New().Overview(header, qs) +} + +func Test(header http.Header, qs url.Values, log *zap.SugaredLogger) ([]byte, error) { + return aslan.New().Test(header, qs) +} + +func Deploy(header http.Header, qs url.Values, log *zap.SugaredLogger) ([]byte, error) { + return aslan.New().Deploy(header, qs) +} + +func Build(header http.Header, qs url.Values, log *zap.SugaredLogger) ([]byte, error) { + return aslan.New().Build(header, qs) } diff --git a/pkg/microservice/picket/core/filter/service/users.go b/pkg/microservice/picket/core/filter/service/users.go new file mode 100644 index 0000000000000000000000000000000000000000..faa6b0fbfb432d16bf19e0a1668e38c5d20eeaf3 --- /dev/null +++ b/pkg/microservice/picket/core/filter/service/users.go @@ -0,0 +1,88 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "net/http" + "net/url" + + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/picket/client/policy" + "github.com/koderover/zadig/pkg/microservice/picket/client/user" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/types" +) + +type DeleteUserResp struct { + Message string `json:"message"` +} + +func DeleteUser(userID string, header http.Header, qs url.Values, _ *zap.SugaredLogger) ([]byte, error) { + _, err := user.New().DeleteUser(userID, header, qs) + if err != nil { + return []byte{}, err + } + return policy.New().DeleteRoleBindings(userID, header, qs) +} + +func SearchUsers(header http.Header, qs url.Values, args *user.SearchArgs, log *zap.SugaredLogger) (*types.UsersResp, error) { + users, err := user.New().SearchUsers(header, qs, args) + if err != nil { + log.Errorf("search users err :%s", err) + return nil, err + } + tmpUids := []string{} + for _, user := range users.Users { + tmpUids = append(tmpUids, user.Uid) + } + + systemMap, err := policy.New().SearchSystemRoleBindings(tmpUids) + if err != nil { + log.Errorf("search system user bindings err :%s", err) + return nil, err + } + + res := &types.UsersResp{ + Users: make([]types.UserInfo, 0), + TotalCount: users.TotalCount, + } + for _, user := range users.Users { + userInfo := types.UserInfo{ + LastLoginTime: user.LastLoginTime, + Uid: user.Uid, + Name: user.Name, + IdentityType: user.IdentityType, + Email: user.Email, + Phone: user.Phone, + Account: user.Account, + APIToken: user.APIToken, + } + if rb, ok := systemMap[user.Uid]; ok { + userInfo.SystemRoleBindings = rb + for _, binding := range rb { + if binding.Role == string(setting.SystemAdmin) { + userInfo.Admin = true + break + } + } + } + res.Users = append(res.Users, userInfo) + } + + return res, nil +} diff --git a/pkg/microservice/picket/core/filter/service/workflow.go b/pkg/microservice/picket/core/filter/service/workflow.go new file mode 100644 index 0000000000000000000000000000000000000000..6664fb5709911fcca304c6c8df031c3d461312aa --- /dev/null +++ b/pkg/microservice/picket/core/filter/service/workflow.go @@ -0,0 +1,143 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "net/http" + "net/url" + + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/koderover/zadig/pkg/microservice/picket/client/aslan" + "github.com/koderover/zadig/pkg/microservice/picket/client/opa" + "github.com/koderover/zadig/pkg/microservice/picket/config" +) + +type rule struct { + method string + endpoint string +} + +func ListWorkflows(header http.Header, qs url.Values, logger *zap.SugaredLogger) ([]byte, error) { + rules := []*rule{{ + method: "/api/aslan/workflow/workflow", + endpoint: "GET", + }} + names, err := getAllowedProjects(header, rules, config.AND, logger) + if err != nil { + logger.Errorf("Failed to get allowed project names, err: %s", err) + return nil, err + } + if len(names) == 0 { + return nil, nil + } + if !(len(names) == 1 && names[0] == "*") { + for _, name := range names { + qs.Add("projects", name) + } + } + return aslan.New().ListWorkflows(header, qs) +} + +func ListWorkflowsV3(header http.Header, qs url.Values, logger *zap.SugaredLogger) ([]byte, error) { + rules := []*rule{{ + method: "/api/aslan/workflow/workflow", + endpoint: "GET", + }} + names, err := getAllowedProjects(header, rules, config.AND, logger) + if err != nil { + logger.Errorf("Failed to get allowed project names, err: %s", err) + return nil, err + } + if len(names) == 0 { + return nil, nil + } + if !(len(names) == 1 && names[0] == "*") { + for _, name := range names { + qs.Add("projects", name) + } + } + return aslan.New().ListWorkflowsV3(header, qs) +} + +func ListTestWorkflows(testName string, header http.Header, qs url.Values, logger *zap.SugaredLogger) ([]byte, error) { + rules := []*rule{{ + method: "/api/aslan/workflow/workflow", + endpoint: "PUT", + }} + names, err := getAllowedProjects(header, rules, config.AND, logger) + if err != nil { + logger.Errorf("Failed to get allowed project names, err: %s", err) + return nil, err + } + if len(names) == 0 { + return nil, nil + } + if !(len(names) == 1 && names[0] == "*") { + for _, name := range names { + qs.Add("projects", name) + } + } + return aslan.New().ListTestWorkflows(testName, header, qs) +} + +//getAllowedProjects +//rulesLogicalOperator@ OR:satisfy any one of rules / AND:satisfy all rules +func getAllowedProjects(headers http.Header, rules []*rule, rulesLogicalOperator config.RulesLogicalOperator, logger *zap.SugaredLogger) (projects []string, err error) { + var res [][]string + for _, v := range rules { + allowedProjects := &allowedProjectsData{} + opaClient := opa.NewDefault() + err := opaClient.Evaluate("rbac.user_allowed_projects", allowedProjects, func() (*opa.Input, error) { + return generateOPAInput(headers, v.method, v.endpoint), nil + }) + if err != nil { + logger.Errorf("opa evaluation failed, err: %s", err) + return nil, err + } + res = append(res, allowedProjects.Result) + } + if rulesLogicalOperator == config.OR { + return union(res), nil + } + return intersect(res), nil +} + +func union(s [][]string) []string { + if len(s) == 0 { + return nil + } + tmp := sets.NewString(s[0]...) + for _, v := range s[1:] { + t := sets.NewString(v...) + tmp = t.Union(tmp) + } + return tmp.List() +} + +func intersect(s [][]string) []string { + if len(s) == 0 { + return nil + } + tmp := sets.NewString(s[0]...) + for _, v := range s[1:] { + t := sets.NewString(v...) + tmp = t.Intersection(tmp) + } + return tmp.List() +} diff --git a/pkg/microservice/picket/core/public/handler/public.go b/pkg/microservice/picket/core/public/handler/public.go new file mode 100644 index 0000000000000000000000000000000000000000..b0406897315a18fb607da9da2150c46a26ea98bd --- /dev/null +++ b/pkg/microservice/picket/core/public/handler/public.go @@ -0,0 +1,160 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "encoding/json" + "net/http" + + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/picket/core/public/service" + "github.com/koderover/zadig/pkg/setting" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" +) + +func CreateWorkflowTask(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + req := new(models.WorkflowTaskArgs) + if err := c.ShouldBindJSON(req); err != nil { + ctx.Err = err + ctx.Logger.Errorf("ShouldBindJSON err:%s", err) + return + } + req.Namespace = req.EnvName + req.RequestMode = setting.RequestModeOpenAPI + + body, err := json.Marshal(req) + if err != nil { + ctx.Err = err + ctx.Logger.Errorf("Marshal err:%s", err) + return + } + res, err := service.CreateWorkflowTask(c.Request.Header, c.Request.URL.Query(), req.WorkflowName, body, ctx.Logger) + if err != nil { + ctx.Err = err + ctx.Logger.Errorf("CreateWorkflowTask err:%s", err) + return + } + // to avoid customer feel confused ，return workflow_name instead of pipline_name + var resp *CreateWorkflowTaskResp + err = json.Unmarshal(res, &resp) + if err != nil { + ctx.Err = err + ctx.Logger.Errorf("Unmarshal err:%s", err) + return + } + resp.WorkflowName = resp.PipelineName + resp.PipelineName = "" + ctx.Resp = resp +} + +type CreateWorkflowTaskResp struct { + ProjectName string `json:"project_name"` + PipelineName string `json:"pipeline_name,omitempty"` + WorkflowName string `json:"workflow_name"` + TaskID int64 `json:"task_id"` +} + +type EndpointResponse struct { + ResultCode int `json:"resultCode"` + ErrorMsg string `json:"errorMsg"` +} + +func CancelWorkflowTask(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + id := c.Param("id") + name := c.Param("name") + statusCode, _ := service.CancelWorkflowTask(c.Request.Header, c.Request.URL.Query(), id, name, ctx.Logger) + var code int + var errorMsg string + if statusCode == http.StatusOK { + code = 0 + errorMsg = "success" + } else if statusCode == http.StatusForbidden { + code = statusCode + errorMsg = "forbidden" + } else { + code = statusCode + errorMsg = "fail" + } + ctx.Resp = EndpointResponse{ + ResultCode: code, + ErrorMsg: errorMsg, + } +} + +func RestartWorkflowTask(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + id := c.Param("id") + name := c.Param("name") + statusCode, _ := service.RestartWorkflowTask(c.Request.Header, c.Request.URL.Query(), id, name, ctx.Logger) + var code int + var errorMsg string + if statusCode == http.StatusOK { + code = 0 + errorMsg = "success" + } else if statusCode == http.StatusForbidden { + code = statusCode + errorMsg = "forbidden" + } else { + code = statusCode + errorMsg = "fail" + } + ctx.Resp = EndpointResponse{ + ResultCode: code, + ErrorMsg: errorMsg, + } +} + +func ListWorkflowTask(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + commitId := c.Query("commitId") + ctx.Resp, ctx.Err = service.ListWorkflowTask(c.Request.Header, c.Request.URL.Query(), commitId, ctx.Logger) +} + +func GetWorkflowDetail(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + id := c.Param("id") + name := c.Param("name") + c.Header("Content-Type", "application/json") + ctx.Resp, ctx.Err = service.GetDetailedWorkflowTask(c.Request.Header, c.Request.URL.Query(), id, name, ctx.Logger) +} + +func ListDelivery(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + productName := c.Query("projectName") + workflowName := c.Query("workflowName") + taskIDStr := c.Query("taskId") + perPageStr := c.Query("perPage") + pageStr := c.Query("page") + ctx.Resp, ctx.Err = service.ListDelivery(c.Request.Header, c.Request.URL.Query(), productName, workflowName, taskIDStr, perPageStr, pageStr, ctx.Logger) +} + +func GetArtifactInfo(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + image := c.Query("image") + ctx.Resp, ctx.Err = service.GetArtifactInfo(c.Request.Header, c.Request.URL.Query(), image, ctx.Logger) +} diff --git a/pkg/middleware/gin/product_name.go b/pkg/microservice/picket/core/public/handler/router.go similarity index 53% rename from pkg/middleware/gin/product_name.go rename to pkg/microservice/picket/core/public/handler/router.go index d66bff4044ac0b1036e68a630eff65419696719a..d3dafc99c4dfb796a78fa847ad229063cbd70dec 100644 --- a/pkg/middleware/gin/product_name.go +++ b/pkg/microservice/picket/core/public/handler/router.go @@ -14,34 +14,24 @@ See the License for the specific language governing permissions and limitations under the License. */ -package gin +package handler import ( - "bytes" - "encoding/json" - "io/ioutil" - "github.com/gin-gonic/gin" - - "github.com/koderover/zadig/pkg/tool/log" ) -type productName struct { - ProductName string `json:"product_name"` -} +type Router struct{} -func StoreProductName(c *gin.Context) { - pn := &productName{} - data, err := c.GetRawData() - if err != nil { - log.Errorf("c.GetRawData() err : %v", err) - return - } - if err = json.Unmarshal(data, pn); err != nil { - log.Errorf("json.Unmarshal err : %v", err) - return +func (*Router) Inject(router *gin.RouterGroup) { + + dev := router.Group("") + { + dev.POST("/workflowTask/create", CreateWorkflowTask) + dev.POST("/workflowTask/id/:id/pipelines/:name/cancel", CancelWorkflowTask) + dev.POST("/workflowTask/id/:id/pipelines/:name/restart", RestartWorkflowTask) + dev.GET("/workflowTask", ListWorkflowTask) + dev.GET("/workflowTask/id/:id/pipelines/:name", GetWorkflowDetail) + dev.GET("/dc/releases", ListDelivery) + dev.GET("/dc/artifact", GetArtifactInfo) } - c.Set("productName", pn.ProductName) - c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) - c.Next() } diff --git a/pkg/microservice/picket/core/public/service/public.go b/pkg/microservice/picket/core/public/service/public.go new file mode 100644 index 0000000000000000000000000000000000000000..e8e8b1c67a759ddea57288b9e257bcc1ff1f7833 --- /dev/null +++ b/pkg/microservice/picket/core/public/service/public.go @@ -0,0 +1,292 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" + "github.com/koderover/zadig/pkg/microservice/picket/client/aslan" + "github.com/koderover/zadig/pkg/tool/log" +) + +type WorkflowTaskImage struct { + Image string `json:"image"` + ServiceName string `json:"service_name"` + RegistryRepo string `json:"registry_repo"` +} + +type WorkflowTaskTarget struct { + Name string `json:"name"` + ServiceType string `json:"service_type"` + Build *WorkflowTaskTargetBuild `json:"build"` +} + +type WorkflowTaskFunctionTestReport struct { + Tests int `json:"tests"` + Successes int `json:"successes"` + Failures int `json:"failures"` + Skips int `json:"skips"` + Errors int `json:"errors"` + DetailUrl string `json:"detail_url"` +} + +type WorkflowTaskTestReport struct { + TestName string `json:"test_name"` + FunctionTestReport *WorkflowTaskFunctionTestReport `json:"function_test_report"` +} + +type WorkflowTaskTargetBuild struct { + Repos []*WorkflowTaskTargetRepo `json:"repos"` +} + +type WorkflowTaskTargetRepo struct { + RepoName string `json:"repo_name"` + Branch string `json:"branch"` + Pr int `json:"pr"` +} + +type WorkflowTaskDetail struct { + WorkflowName string `json:"workflow_name"` + EnvName string `json:"env_name"` + Targets []*WorkflowTaskTarget `json:"targets"` + Images []*WorkflowTaskImage `json:"images"` + TestReports []*WorkflowTaskTestReport `json:"test_reports"` + Status string `json:"status"` +} + +func CreateWorkflowTask(header http.Header, qs url.Values, workflowName string, body []byte, _ *zap.SugaredLogger) ([]byte, error) { + return aslan.New().CreateWorkflowTask(header, qs, body, workflowName) +} + +func CancelWorkflowTask(header http.Header, qs url.Values, id string, name string, _ *zap.SugaredLogger) (int, error) { + return aslan.New().CancelWorkflowTask(header, qs, id, name) +} + +func RestartWorkflowTask(header http.Header, qs url.Values, id string, name string, _ *zap.SugaredLogger) (int, error) { + return aslan.New().RestartWorkflowTask(header, qs, id, name) +} + +func ListWorkflowTask(header http.Header, qs url.Values, commitId string, _ *zap.SugaredLogger) ([]byte, error) { + return aslan.New().ListWorkflowTask(header, qs, commitId) +} + +func getTargets(task *task.Task) ([]*WorkflowTaskTarget, error) { + ret := make([]*WorkflowTaskTarget, 0) + if task.WorkflowArgs != nil { + for _, singleTarget := range task.WorkflowArgs.Target { + target := &WorkflowTaskTarget{ + Name: singleTarget.ServiceName, // return service name instead of container name + ServiceType: singleTarget.ServiceType, + Build: &WorkflowTaskTargetBuild{ + Repos: make([]*WorkflowTaskTargetRepo, 0), + }, + } + if singleTarget.Build != nil { + for _, repo := range singleTarget.Build.Repos { + target.Build.Repos = append(target.Build.Repos, &WorkflowTaskTargetRepo{ + RepoName: repo.RepoName, + Branch: repo.Branch, + Pr: repo.PR, + }) + } + } + ret = append(ret, target) + } + } + return ret, nil +} + +func getImages(task *task.Task) ([]*WorkflowTaskImage, error) { + ret := make([]*WorkflowTaskImage, 0) + for _, stages := range task.Stages { + if stages.TaskType != config.TaskBuild || stages.Status != config.StatusPassed { + continue + } + for _, subTask := range stages.SubTasks { + buildInfo, err := base.ToBuildTask(subTask) + if err != nil { + log.Errorf("get buildInfo ToBuildTask failed ! err:%s", err) + return nil, err + } + + WorkflowTaskImage := &WorkflowTaskImage{ + Image: buildInfo.JobCtx.Image, + ServiceName: buildInfo.ServiceName, + } + if buildInfo.DockerBuildStatus != nil { + WorkflowTaskImage.RegistryRepo = buildInfo.DockerBuildStatus.RegistryRepo + } + ret = append(ret, WorkflowTaskImage) + } + } + return ret, nil +} + +func getTestReports(task *task.Task) ([]*WorkflowTaskTestReport, error) { + ret := make([]*WorkflowTaskTestReport, 0) + + for testName, testData := range task.TestReports { + tr := new(models.TestReport) + bs, err := json.Marshal(testData) + if err != nil { + return nil, err + } + err = json.Unmarshal(bs, tr) + if err != nil { + return nil, err + } + // TODO currently only return function test data + if tr.FunctionTestSuite == nil { + continue + } + + detailUrl := fmt.Sprintf("/v1/projects/detail/%s/pipelines/multi/testcase/%s/%d/test/%s?is_workflow=1&service_name=%s&test_type=function", + task.ProductName, task.PipelineName, task.TaskID, fmt.Sprintf("%s-%d-test", task.PipelineName, task.TaskID), testName) + + fts := tr.FunctionTestSuite + ret = append(ret, &WorkflowTaskTestReport{ + TestName: testName, + FunctionTestReport: &WorkflowTaskFunctionTestReport{ + Tests: fts.Tests, + Successes: fts.Successes, + Failures: fts.Failures, + Skips: fts.Skips, + Errors: fts.Errors, + DetailUrl: detailUrl, + }, + }) + } + + return ret, nil +} + +func GetDetailedWorkflowTask(header http.Header, qs url.Values, taskID, name string, _ *zap.SugaredLogger) ([]byte, error) { + body, err := aslan.New().GetDetailedWorkflowTask(header, qs, taskID, name) + if err != nil { + return nil, err + } + var workflowTask *task.Task = &task.Task{} + if err = json.Unmarshal(body, workflowTask); err != nil { + return nil, fmt.Errorf("failed to unmarshal workflowTask err:%s", err) + } + + resp := &WorkflowTaskDetail{ + WorkflowName: workflowTask.PipelineName, + EnvName: workflowTask.WorkflowArgs.EnvName, + Status: string(workflowTask.Status), + } + + resp.Targets, err = getTargets(workflowTask) + if err != nil { + return nil, err + } + + resp.Images, err = getImages(workflowTask) + if err != nil { + return nil, err + } + + resp.TestReports, err = getTestReports(workflowTask) + if err != nil { + return nil, err + } + + return json.Marshal(resp) +} + +func ListDelivery(header http.Header, qs url.Values, productName string, workflowName string, taskId string, perPage string, page string, _ *zap.SugaredLogger) ([]byte, error) { + return aslan.New().ListDelivery(header, qs, productName, workflowName, taskId, perPage, page) +} + +type DeliveryArtifact struct { + Name string `json:"name"` + Type string `json:"type"` + Source string `json:"source"` + Image string `json:"image,omitempty"` + ImageHash string `json:"image_hash,omitempty"` + ImageTag string `json:"image_tag"` + ImageDigest string `json:"image_digest,omitempty"` + ImageSize int64 `json:"image_size,omitempty"` + Architecture string `json:"architecture,omitempty"` + Os string `json:"os,omitempty"` + PackageFileLocation string `json:"package_file_location,omitempty"` + PackageStorageURI string `json:"package_storage_uri,omitempty"` + CreatedBy string `json:"created_by"` + CreatedTime int64 `json:"created_time"` +} + +type DeliveryActivity struct { + Type string `json:"type"` + Content string `json:"content,omitempty"` + URL string `json:"url,omitempty"` + Commits []*ActivityCommit `json:"commits,omitempty"` + Issues []string `json:"issues,omitempty"` + Namespace string `json:"namespace,omitempty"` + EnvName string `json:"env_name,omitempty"` + PublishHosts []string `json:"publish_hosts,omitempty"` + PublishNamespaces []string `json:"publish_namespaces,omitempty"` + RemoteFileKey string `json:"remote_file_key,omitempty"` + DistStorageURL string `json:"dist_storage_url,omitempty"` + SrcStorageURL string `json:"src_storage_url,omitempty"` + StartTime int64 `json:"start_time,omitempty"` + EndTime int64 `json:"end_time,omitempty"` + CreatedBy string `json:"created_by"` + CreatedTime int64 `json:"created_time"` +} + +type ActivityCommit struct { + Address string `json:"address"` + Source string `json:"source,omitempty"` + RepoOwner string `json:"repo_owner"` + RepoName string `json:"repo_name"` + Branch string `json:"branch"` + PR int `json:"pr,omitempty"` + Tag string `json:"tag,omitempty"` + CommitID string `json:"commit_id,omitempty"` + CommitMessage string `json:"commit_message,omitempty"` + AuthorName string `json:"author_name,omitempty"` +} + +type DeliveryArtifactInfo struct { + *DeliveryArtifact + DeliveryActivities []*DeliveryActivity `json:"activities"` + DeliveryActivitiesMap map[string][]*DeliveryActivity `json:"sortedActivities,omitempty"` +} + +func GetArtifactInfo(header http.Header, qs url.Values, image string, _ *zap.SugaredLogger) (*DeliveryArtifactInfo, error) { + artifactID, err := aslan.New().GetArtifactByImage(header, qs, image) + if err != nil { + return nil, fmt.Errorf("failed to get artifact by image err:%s", err) + } + + body, err := aslan.New().GetArtifactInfo(header, qs, string(artifactID)) + var deliveryArtifactInfo *DeliveryArtifactInfo + if err = json.Unmarshal(body, &deliveryArtifactInfo); err != nil { + return nil, fmt.Errorf("failed to unmarshal deliveryArtifactInfo err:%s", err) + } + + return deliveryArtifactInfo, nil +} diff --git a/pkg/microservice/picket/server/rest/router.go b/pkg/microservice/picket/server/rest/router.go index 6eece402f47de25364911b27309f55b1d4201b67..ba02b80ebc35067d01babe4f13a73d399f466361 100644 --- a/pkg/microservice/picket/server/rest/router.go +++ b/pkg/microservice/picket/server/rest/router.go @@ -20,14 +20,24 @@ import ( "github.com/gin-gonic/gin" evaluationhandler "github.com/koderover/zadig/pkg/microservice/picket/core/evaluation/handler" + filterhandler "github.com/koderover/zadig/pkg/microservice/picket/core/filter/handler" + publichandler "github.com/koderover/zadig/pkg/microservice/picket/core/public/handler" ) func (s *engine) injectRouterGroup(router *gin.RouterGroup) { for _, r := range []injector{ new(evaluationhandler.Router), + new(filterhandler.Router), } { r.Inject(router.Group("/api/v1")) } + + for _, r := range []injector{ + new(publichandler.Router), + } { + r.Inject(router.Group("/public-api/v1")) + } + } type injector interface { diff --git a/pkg/microservice/podexec/config/config.go b/pkg/microservice/podexec/config/config.go index c75b077e0a22a39e8093a2624e474044a5849546..782d8b451e31106324cf989b153bf76be786736f 100644 --- a/pkg/microservice/podexec/config/config.go +++ b/pkg/microservice/podexec/config/config.go @@ -17,16 +17,9 @@ limitations under the License. package config import ( - "github.com/spf13/viper" - configbase "github.com/koderover/zadig/pkg/config" - "github.com/koderover/zadig/pkg/setting" ) -func PoetryAPIRootKey() string { - return viper.GetString(setting.ENVPoetryAPIRootKey) -} - func HubServerAddr() string { return configbase.HubServerServiceAddress() } diff --git a/pkg/microservice/podexec/core/service/auth.go b/pkg/microservice/podexec/core/service/auth.go deleted file mode 100644 index e45b87fef8870eea3dc1971f01cd11295f967ab4..0000000000000000000000000000000000000000 --- a/pkg/microservice/podexec/core/service/auth.go +++ /dev/null @@ -1,150 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package service - -import ( - "encoding/json" - "net/http" - "strconv" - "strings" - - "github.com/gorilla/mux" - "go.uber.org/zap" - "k8s.io/apimachinery/pkg/util/sets" - - configbase "github.com/koderover/zadig/pkg/config" - "github.com/koderover/zadig/pkg/microservice/podexec/config" - "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/poetry" - "github.com/koderover/zadig/pkg/tool/log" - "github.com/koderover/zadig/pkg/types/permission" -) - -func AuthMiddleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - url := r.URL.Path - if strings.Contains("/api/health/", url) { - next.ServeHTTP(w, r) - return - } - - authorization := r.Header.Get(setting.AuthorizationHeader) - if authorization != "" { - if strings.Contains(authorization, setting.RootAPIKey) { - token := strings.Split(authorization, " ") - if len(token) == 2 && token[1] == config.PoetryAPIRootKey() { - next.ServeHTTP(w, r) - } - return - } - - if strings.Contains(authorization, setting.UserAPIKey) { - token := strings.Split(authorization, " ") - if len(token) == 2 { - //根据token获取用户 - userInfo, err := poetry.GetUserDetailByToken(configbase.PoetryServiceAddress(), token[1]) - if err != nil { - w.WriteHeader(http.StatusUnauthorized) - _ = json.NewEncoder(w).Encode(&EndpointResponse{ResultCode: http.StatusUnauthorized, ErrorMsg: "auth failed"}) - return - } - - r.Header.Set("userId", strconv.Itoa(userInfo.ID)) - r.Header.Set("isSuperUser", strconv.FormatBool(userInfo.IsSuperUser)) - next.ServeHTTP(w, r) - } - return - } - } else if token, err := r.Cookie("TOKEN"); err == nil { - userInfo, err := poetry.GetUserDetailByToken(configbase.PoetryServiceAddress(), token.Value) - if err != nil { - log.Errorf("PoetryRequest err:%v", err) - w.WriteHeader(http.StatusUnauthorized) - _ = json.NewEncoder(w).Encode(&EndpointResponse{ResultCode: http.StatusUnauthorized, ErrorMsg: "auth failed"}) - return - } - - r.Header.Set("userId", strconv.Itoa(userInfo.ID)) - r.Header.Set("isSuperUser", strconv.FormatBool(userInfo.IsSuperUser)) - next.ServeHTTP(w, r) - } - }) -} - -func PermissionMiddleware(next http.Handler) http.Handler { - logger := log.SugaredLogger() - - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - url := r.URL.Path - if strings.Contains("/api/health/", url) { - next.ServeHTTP(w, r) - return - } - pathParams := mux.Vars(r) - productName := pathParams["productName"] - userIDStr := r.Header.Get("userId") - isSuperUserStr := r.Header.Get("isSuperUser") - isSuperUser, parseErr := strconv.ParseBool(isSuperUserStr) - if isSuperUser { - next.ServeHTTP(w, r) - return - } - userID, atoiErr := strconv.Atoi(userIDStr) - if parseErr != nil || atoiErr != nil { - w.WriteHeader(http.StatusForbidden) - _ = json.NewEncoder(w).Encode(&EndpointResponse{ResultCode: http.StatusForbidden, ErrorMsg: "forbidden!"}) - return - } - - queryList := r.URL.Query() - clusterID := queryList.Get("clusterId") - if clusterID == "" { - isHaveTestEnvManagePermission := PoetryRequestPermission(userID, permission.TestEnvManageUUID, productName, logger) - if isHaveTestEnvManagePermission { - next.ServeHTTP(w, r) - return - } - } else { - isHaveProdEnvManagePermission := PoetryRequestPermission(userID, permission.ProdEnvManageUUID, productName, logger) - if isHaveProdEnvManagePermission { - next.ServeHTTP(w, r) - return - } - } - - w.WriteHeader(http.StatusForbidden) - _ = json.NewEncoder(w).Encode(&EndpointResponse{ResultCode: http.StatusForbidden, ErrorMsg: "forbidden!"}) - }) -} - -func PoetryRequestPermission(userID int, permissionUUID, productName string, logger *zap.SugaredLogger) bool { - poetryClient := poetry.New(configbase.PoetryServiceAddress(), config.PoetryAPIRootKey()) - if poetryClient.HasOperatePermission(productName, permissionUUID, userID, false, logger) { - return true - } - - uuids, err := poetryClient.GetUserPermissionUUIDs(setting.RoleUserID, "", logger) - if err != nil { - log.Errorf("GetUserPermissionUUIDs error: %v", err) - return false - } - if sets.NewString(uuids...).Has(permissionUUID) { - return true - } - - return false -} diff --git a/pkg/microservice/podexec/core/service/ws_terminal.go b/pkg/microservice/podexec/core/service/ws_terminal.go index dfb805ecc0e128001136a0041fa956bf02126314..69b4b5230ce9801839ad7b008b5decfc5c9063ab 100644 --- a/pkg/microservice/podexec/core/service/ws_terminal.go +++ b/pkg/microservice/podexec/core/service/ws_terminal.go @@ -35,6 +35,8 @@ import ( "k8s.io/client-go/tools/remotecommand" conf "github.com/koderover/zadig/pkg/microservice/podexec/config" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/kube/wrapper" "github.com/koderover/zadig/pkg/tool/log" ) @@ -156,11 +158,21 @@ func ValidatePod(kubeClient kubernetes.Interface, namespace, podName, containerN if pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed { return false, fmt.Errorf("cannot exec into a container in a completed pod; current phase is %s", pod.Status.Phase) } + for _, c := range pod.Spec.Containers { if containerName == c.Name { return true, nil } } + + if wrapper.CheckEphemeralContainerFieldExist(&pod.Spec) { + for _, c := range pod.Spec.EphemeralContainers { + if containerName == c.Name { + return true, nil + } + } + } + return false, fmt.Errorf("pod has no container '%s'", containerName) } @@ -206,7 +218,7 @@ func NewKubeOutClusterClient(clusterID string) (kubernetes.Interface, *rest.Conf var config *rest.Config var err error - if clusterID == "" { + if clusterID == "" || clusterID == setting.LocalClusterID { kubeConfig := "" config, err = clientcmd.BuildConfigFromFlags("", kubeConfig) if err != nil { diff --git a/pkg/microservice/podexec/server/server.go b/pkg/microservice/podexec/server/server.go index 55f12b9299af14bb3bfb515604ccda115c5015d2..c326fe3b8e1f8182b760de645798917742ade431 100644 --- a/pkg/microservice/podexec/server/server.go +++ b/pkg/microservice/podexec/server/server.go @@ -42,9 +42,7 @@ func Serve(ctx context.Context) error { router.HandleFunc("/api/health", func(w http.ResponseWriter, r *http.Request) { _ = json.NewEncoder(w).Encode(map[string]string{"message": "success"}) }) - router.HandleFunc("/api/{productName}/{namespace}/{podName}/{containerName}/podExec", service.ServeWs) - router.Use(service.AuthMiddleware) - router.Use(service.PermissionMiddleware) + router.HandleFunc("/api/{productName}/{namespace}/{podName}/{containerName}/podExec/{envName}", service.ServeWs) server := &http.Server{ Addr: "0.0.0.0:27000", diff --git a/pkg/microservice/policy/core/handler/bundle.go b/pkg/microservice/policy/core/handler/bundle.go index 7f3e1b7610378b1861eb02dd0f1230b6298d4d3b..71bd6c7c8c718d9b5d32009338336fab79cc4853 100644 --- a/pkg/microservice/policy/core/handler/bundle.go +++ b/pkg/microservice/policy/core/handler/bundle.go @@ -24,9 +24,16 @@ import ( "github.com/koderover/zadig/pkg/config" "github.com/koderover/zadig/pkg/microservice/policy/core/service/bundle" + "github.com/koderover/zadig/pkg/tool/log" ) func DownloadBundle(c *gin.Context) { + if err := bundle.GenerateOPABundle(); err != nil { + log.Errorf("Failed to generate OPA bundle, err: %s", err) + c.String(http.StatusInternalServerError, "bundle generation failure, err: %s", err) + return + } + revision := bundle.GetRevision() matching := c.GetHeader("If-None-Match") if revision != "" && revision == matching { diff --git a/pkg/microservice/policy/core/handler/healthz.go b/pkg/microservice/policy/core/handler/healthz.go new file mode 100644 index 0000000000000000000000000000000000000000..860a2e170cfc45912e09857142a7c6ef97f449d3 --- /dev/null +++ b/pkg/microservice/policy/core/handler/healthz.go @@ -0,0 +1,30 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/policy/core" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" +) + +func Healthz(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + ctx.Err = core.Healthz(c.Request.Context()) +} diff --git a/pkg/microservice/policy/core/handler/permission.go b/pkg/microservice/policy/core/handler/permission.go new file mode 100644 index 0000000000000000000000000000000000000000..5646fafac984b1ede3f6f5347c26179dc6d2ce6d --- /dev/null +++ b/pkg/microservice/policy/core/handler/permission.go @@ -0,0 +1,59 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/policy/core/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +func GetUserRules(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.GetUserRules(ctx.UserID, ctx.Logger) +} + +func GetUserRulesByProject(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.GetUserRulesByProject(ctx.UserID, c.Param("name"), ctx.Logger) +} + +type GetUserResourcesPermissionReq struct { + ProjectName string `json:"project_name" form:"project_name"` + Uid string `json:"uid" form:"uid"` + Resources []string `json:"resources" form:"resources"` + ResourceType string `json:"resource_type" form:"resource_type"` +} + +func GetUserResourcesPermission(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + req := new(GetUserResourcesPermissionReq) + + if err := c.ShouldBindJSON(req); err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + ctx.Resp, ctx.Err = service.GetResourcesPermission(req.Uid, req.ProjectName, req.ResourceType, req.Resources, ctx.Logger) +} diff --git a/pkg/microservice/policy/core/handler/policy.go b/pkg/microservice/policy/core/handler/policy.go new file mode 100644 index 0000000000000000000000000000000000000000..d19218ec69c4511abfb018a2b6c7b8136f95403c --- /dev/null +++ b/pkg/microservice/policy/core/handler/policy.go @@ -0,0 +1,248 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/policy/core/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +type deletePoliciesArgs struct { + Names []string `json:"names"` +} +type createPoliciesArgs struct { + Policies []*service.Policy `json:"policies"` +} + +func CreatePolicies(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := &createPoliciesArgs{} + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return + } + + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("projectName is empty") + return + } + ctx.Err = service.CreatePolicies(projectName, args.Policies, ctx.Logger) +} + +func UpdatePolicy(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := &service.Policy{} + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return + } + + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("projectName is empty") + return + } + name := c.Param("name") + args.Name = name + + ctx.Err = service.UpdatePolicy(projectName, args, ctx.Logger) +} + +func UpdateOrCreatePolicy(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := &service.Policy{} + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return + } + + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("projectName is empty") + return + } + args.Name = c.Param("name") + + ctx.Err = service.UpdateOrCreatePolicy(projectName, args, ctx.Logger) +} + +func UpdatePublicPolicy(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := &service.Policy{} + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return + } + name := c.Param("name") + args.Name = name + ctx.Err = service.UpdatePolicy(service.PresetScope, args, ctx.Logger) +} + +func UpdateOrCreatePublicPolicy(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := &service.Policy{} + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return + } + name := c.Param("name") + args.Name = name + ctx.Err = service.UpdateOrCreatePolicy(service.PresetScope, args, ctx.Logger) +} + +func ListPolicies(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + projectName := c.Query("projectName") + + ctx.Resp, ctx.Err = service.ListPolicies(projectName, ctx.Logger) +} + +func GetPolicy(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + projectName := c.Query("projectName") + + ctx.Resp, ctx.Err = service.GetPolicy(projectName, c.Param("name"), ctx.Logger) +} + +func GetPolicies(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.GetPolicies(c.Query("names"), ctx.Logger) +} + +func CreatePublicPolicy(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := &service.Policy{} + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return + } + + ctx.Err = service.CreatePolicy(service.PresetScope, args, ctx.Logger) +} + +func ListPublicPolicies(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.ListPolicies(service.PresetScope, ctx.Logger) + return +} + +func GetPublicPolicy(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.GetPolicy(service.PresetScope, c.Param("name"), ctx.Logger) +} + +func DeletePolicy(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + name := c.Param("name") + projectName := c.Query("projectName") + + ctx.Err = service.DeletePolicy(name, projectName, ctx.Logger) +} + +func DeletePolicies(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + projectName := c.Query("projectName") + + args := &deletePoliciesArgs{} + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return + } + + ctx.Err = service.DeletePolicies(args.Names, projectName, ctx.Logger) +} + +func DeletePublicPolicies(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + name := c.Param("name") + ctx.Err = service.DeletePolicy(name, service.PresetScope, ctx.Logger) + return +} + +func CreateSystemPolicy(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := &service.Policy{} + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return + } + + ctx.Err = service.CreatePolicy(service.SystemScope, args, ctx.Logger) +} + +func UpdateOrCreateSystemPolicy(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := &service.Policy{} + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return + } + name := c.Param("name") + args.Name = name + ctx.Err = service.UpdateOrCreatePolicy(service.SystemScope, args, ctx.Logger) +} + +func ListSystemPolicies(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.ListPolicies(service.SystemScope, ctx.Logger) + return +} + +func DeleteSystemPolicy(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + name := c.Param("name") + ctx.Err = service.DeletePolicy(name, service.SystemScope, ctx.Logger) + return +} diff --git a/pkg/microservice/policy/core/handler/policy_binding.go b/pkg/microservice/policy/core/handler/policy_binding.go new file mode 100644 index 0000000000000000000000000000000000000000..2fd0e172c6fd758831fd7053d82ada7ed102aba7 --- /dev/null +++ b/pkg/microservice/policy/core/handler/policy_binding.go @@ -0,0 +1,108 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/policy/core/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +type deletePolicyBindingsArgs struct { + Names []string `json:"names"` +} + +func UpdatePolicyBinding(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("projectName is empty") + return + } + args := &service.PolicyBinding{} + if err := c.ShouldBindJSON(&args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("bind json fail") + } + ctx.Err = service.UpdateOrCreatePolicyBinding(projectName, args, ctx.Logger) +} + +func CreatePolicyBinding(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("projectName is empty") + return + } + + args := make([]*service.PolicyBinding, 0) + + if err := c.ShouldBindJSON(&args); err != nil { + ctx.Err = err + return + } + + ctx.Err = service.CreatePolicyBindings(projectName, args, ctx.Logger) +} + +func ListPolicyBindings(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("projectName is empty") + return + } + + ctx.Resp, ctx.Err = service.ListPolicyBindings(projectName, "", ctx.Logger) +} + +func DeletePolicyBinding(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + name := c.Param("name") + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("projectName is empty") + return + } + + ctx.Err = service.DeletePolicyBinding(name, projectName, ctx.Logger) +} + +func DeletePolicyBindings(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + projectName := c.Query("projectName") + userID := c.Query("userID") + + args := &deletePolicyBindingsArgs{} + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return + } + + ctx.Err = service.DeletePolicyBindings(args.Names, projectName, userID, ctx.Logger) +} diff --git a/pkg/microservice/policy/core/handler/policy_meta_registration.go b/pkg/microservice/policy/core/handler/policy_meta_registration.go new file mode 100644 index 0000000000000000000000000000000000000000..8de9e877fabc1f271fff4be578841d71f4ac0ebf --- /dev/null +++ b/pkg/microservice/policy/core/handler/policy_meta_registration.go @@ -0,0 +1,32 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/policy/core/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" +) + +func GetPolicyRegistrationDefinitions(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + scope := c.Query("scope") + envType := c.Query("env_type") + ctx.Resp, ctx.Err = service.GetPolicyRegistrationDefinitions(scope, envType, ctx.Logger) +} diff --git a/pkg/microservice/policy/core/handler/role.go b/pkg/microservice/policy/core/handler/role.go index 25d51344095e67c556f08c8f9a20d50d5e488a0a..5c657a2a7078f50399bc0616e2e57a6d0d426bec 100644 --- a/pkg/microservice/policy/core/handler/role.go +++ b/pkg/microservice/policy/core/handler/role.go @@ -20,10 +20,15 @@ import ( "github.com/gin-gonic/gin" "github.com/koderover/zadig/pkg/microservice/policy/core/service" + "github.com/koderover/zadig/pkg/setting" internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" ) +type deleteRolesArgs struct { + Names []string `json:"names"` +} + func CreateRole(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() @@ -43,7 +48,48 @@ func CreateRole(c *gin.Context) { ctx.Err = service.CreateRole(projectName, args, ctx.Logger) } -func CreateGlobalRole(c *gin.Context) { +func UpdateRole(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := &service.Role{} + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return + } + + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("projectName is empty") + return + } + name := c.Param("name") + args.Name = name + + ctx.Err = service.UpdateRole(projectName, args, ctx.Logger) +} + +func UpdateOrCreateRole(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := &service.Role{} + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return + } + + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("projectName is empty") + return + } + args.Name = c.Param("name") + + ctx.Err = service.UpdateOrCreateRole(projectName, args, ctx.Logger) +} + +func UpdatePresetRole(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() @@ -52,6 +98,158 @@ func CreateGlobalRole(c *gin.Context) { ctx.Err = err return } + name := c.Param("name") + args.Name = name + ctx.Err = service.UpdateRole(service.PresetScope, args, ctx.Logger) +} + +func UpdateOrCreatePresetRole(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := &service.Role{} + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return + } + name := c.Param("name") + args.Name = name + ctx.Err = service.UpdateOrCreateRole(service.PresetScope, args, ctx.Logger) +} + +func ListRoles(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("args projectName can't be empty") + return + } + + ctx.Resp, ctx.Err = service.ListRoles(projectName, ctx.Logger) +} + +func GetRole(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("args projectName can't be empty") + return + } - ctx.Err = service.CreateRole("", args, ctx.Logger) + ctx.Resp, ctx.Err = service.GetRole(projectName, c.Param("name"), ctx.Logger) +} + +func CreatePresetRole(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := &service.Role{} + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return + } + args.Type = setting.ResourceTypeSystem + ctx.Err = service.CreateRole(service.PresetScope, args, ctx.Logger) +} + +func ListPresetRoles(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.ListRoles(service.PresetScope, ctx.Logger) + return +} + +func GetPresetRole(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.GetRole(service.PresetScope, c.Param("name"), ctx.Logger) +} + +func DeleteRole(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + name := c.Param("name") + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("args projectName can't be empty") + return + } + + ctx.Err = service.DeleteRole(name, projectName, ctx.Logger) +} + +func DeleteRoles(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("args projectName can't be empty") + return + } + + args := &deleteRolesArgs{} + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return + } + + ctx.Err = service.DeleteRoles(args.Names, projectName, ctx.Logger) +} + +func DeletePresetRole(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + name := c.Param("name") + ctx.Err = service.DeleteRole(name, service.PresetScope, ctx.Logger) + return +} + +func CreateSystemRole(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := &service.Role{} + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return + } + ctx.Err = service.CreateRole(service.SystemScope, args, ctx.Logger) +} + +func UpdateOrCreateSystemRole(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := &service.Role{} + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return + } + name := c.Param("name") + args.Name = name + ctx.Err = service.UpdateOrCreateRole(service.SystemScope, args, ctx.Logger) +} + +func ListSystemRoles(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.ListRoles(service.SystemScope, ctx.Logger) + return +} + +func DeleteSystemRole(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + name := c.Param("name") + ctx.Err = service.DeleteRole(name, service.SystemScope, ctx.Logger) + return } diff --git a/pkg/microservice/policy/core/handler/role_binding.go b/pkg/microservice/policy/core/handler/role_binding.go index 33b73e8fcda57f83287a2a9c4145eee63387fa20..cb9fb51b3bf728ebbf16d992d31f997f68fc88ab 100644 --- a/pkg/microservice/policy/core/handler/role_binding.go +++ b/pkg/microservice/policy/core/handler/role_binding.go @@ -24,6 +24,26 @@ import ( e "github.com/koderover/zadig/pkg/tool/errors" ) +type deleteRoleBindingsArgs struct { + Names []string `json:"names"` +} + +func UpdateRoleBinding(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("projectName is empty") + return + } + args := &service.RoleBinding{} + if err := c.ShouldBindJSON(&args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("bind json fail %s") + } + ctx.Err = service.UpdateOrCreateRoleBinding(projectName, args, ctx.Logger) +} + func CreateRoleBinding(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() @@ -34,11 +54,201 @@ func CreateRoleBinding(c *gin.Context) { return } + args := make([]*service.RoleBinding, 0) + if c.Query("bulk") == "true" { + if err := c.ShouldBindJSON(&args); err != nil { + ctx.Err = err + return + } + } else { + rb := &service.RoleBinding{} + if err := c.ShouldBindJSON(rb); err != nil { + ctx.Err = err + return + } + args = append(args, rb) + } + + ctx.Err = service.CreateRoleBindings(projectName, args, ctx.Logger) +} + +func ListRoleBindings(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("projectName is empty") + return + } + + ctx.Resp, ctx.Err = service.ListRoleBindings(projectName, "", ctx.Logger) +} + +func DeleteRoleBinding(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + name := c.Param("name") + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("projectName is empty") + return + } + + ctx.Err = service.DeleteRoleBinding(name, projectName, ctx.Logger) +} + +func DeleteRoleBindings(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + projectName := c.Query("projectName") + userID := c.Query("userID") + + args := &deleteRoleBindingsArgs{} + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return + } + + ctx.Err = service.DeleteRoleBindings(args.Names, projectName, userID, ctx.Logger) +} + +func UpdateRoleBindings(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + projectName := c.Query("projectName") + if projectName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("projectName is empty") + return + } + userID := c.Query("userID") + if userID == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("userID is empty") + return + } + args := make([]*service.RoleBinding, 0) + if err := c.ShouldBindJSON(&args); err != nil { + ctx.Err = err + return + } + ctx.Err = service.UpdateRoleBindings(projectName, args, c.Query("userID"), ctx.Logger) +} + +func UpdateSystemRoleBindings(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + userID := c.Query("userID") + if userID == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("userID is empty") + return + } + + args := make([]*service.RoleBinding, 0) + if err := c.ShouldBindJSON(&args); err != nil { + ctx.Err = err + return + } + var userId string + for i, v := range args { + if i == 0 { + userId = v.UID + } + if v.UID == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("some roleBinding's uid is empty") + return + } + + if v.UID != userId { + ctx.Err = e.ErrInvalidParam.AddDesc("roleBindings' uid is not consistent") + return + } + } + // TODO: mouuii args can be empty slice , this will delete admin binding , try to improve + if userId == "" { + userId = userID + } + ctx.Err = service.UpdateRoleBindings(service.SystemScope, args, userId, ctx.Logger) +} + +func DeleteSystemRoleBinding(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + name := c.Param("name") + ctx.Err = service.DeleteRoleBinding(name, service.SystemScope, ctx.Logger) +} + +func CreateSystemRoleBinding(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := &service.RoleBinding{} + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return + } + + args.Preset = false + + ctx.Err = service.CreateRoleBindings(service.SystemScope, []*service.RoleBinding{args}, ctx.Logger) +} + +func CreateOrUpdateSystemRoleBinding(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + args := &service.RoleBinding{} if err := c.ShouldBindJSON(args); err != nil { ctx.Err = err return } - ctx.Err = service.CreateRoleBinding(projectName, args, ctx.Logger) + args.Preset = false + + ctx.Err = service.CreateOrUpdateSystemRoleBinding(service.SystemScope, args, ctx.Logger) +} + +func ListSystemRoleBindings(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.ListRoleBindings(service.SystemScope, "", ctx.Logger) +} + +type SearchSystemRoleBindingArgs struct { + Uids []string `json:"uids"` +} + +func SearchSystemRoleBinding(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(SearchSystemRoleBindingArgs) + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + + ctx.Resp, ctx.Err = service.SearchSystemRoleBindings(args.Uids, ctx.Logger) +} + +func ListUserBindings(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + uid := c.Query("uid") + if uid == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("uid is empty") + return + } + projectName := c.Query("projectName") + if projectName == "" { + projectName = service.SystemScope + } + + ctx.Resp, ctx.Err = service.ListRoleBindings(projectName, uid, ctx.Logger) } diff --git a/pkg/microservice/policy/core/handler/router.go b/pkg/microservice/policy/core/handler/router.go index 32cd3db6c5496cc00e4eb7750356cf22a171f126..96a9efb4d70a896e922ea217a21c54ad83501cf5 100644 --- a/pkg/microservice/policy/core/handler/router.go +++ b/pkg/microservice/policy/core/handler/router.go @@ -26,20 +26,96 @@ func (*Router) Inject(router *gin.RouterGroup) { roles := router.Group("roles") { roles.POST("", CreateRole) + roles.POST("/bulk-delete", DeleteRoles) + roles.PATCH("/:name", UpdateRole) + roles.PUT("/:name", UpdateOrCreateRole) + roles.GET("", ListRoles) + roles.GET("/:name", GetRole) + roles.DELETE("/:name", DeleteRole) } - globalRoles := router.Group("global-roles") + policies := router.Group("policies") { - globalRoles.POST("", CreateGlobalRole) + policies.POST("", CreatePolicies) + policies.POST("/bulk-delete", DeletePolicies) + policies.PATCH("/:name", UpdatePolicy) + policies.PUT("/:name", UpdateOrCreatePolicy) + policies.GET("", ListPolicies) + policies.GET("/:name", GetPolicy) + policies.GET("/bulk", GetPolicies) + policies.DELETE("/:name", DeletePolicy) + } + + presetRoles := router.Group("preset-roles") + { + presetRoles.POST("", CreatePresetRole) + presetRoles.GET("", ListPresetRoles) + presetRoles.GET("/:name", GetPresetRole) + presetRoles.PATCH("/:name", UpdatePresetRole) + presetRoles.PUT("/:name", UpdateOrCreatePresetRole) + presetRoles.DELETE("/:name", DeletePresetRole) + } + + systemRoles := router.Group("system-roles") + { + systemRoles.POST("", CreateSystemRole) + systemRoles.PUT("/:name", UpdateOrCreateSystemRole) + systemRoles.GET("", ListSystemRoles) + systemRoles.DELETE("/:name", DeleteSystemRole) } roleBindings := router.Group("rolebindings") { roleBindings.POST("", CreateRoleBinding) + roleBindings.PUT("/:name", UpdateRoleBinding) + roleBindings.GET("", ListRoleBindings) + roleBindings.DELETE("/:name", DeleteRoleBinding) + roleBindings.POST("/bulk-delete", DeleteRoleBindings) + roleBindings.POST("/update", UpdateRoleBindings) + } + + policyBindings := router.Group("policybindings") + { + policyBindings.POST("", CreatePolicyBinding) + policyBindings.PUT("/:name", UpdatePolicyBinding) + policyBindings.GET("", ListPolicyBindings) + policyBindings.DELETE("/:name", DeletePolicyBinding) + policyBindings.POST("/bulk-delete", DeletePolicyBindings) + } + + systemRoleBindings := router.Group("system-rolebindings") + { + systemRoleBindings.POST("", CreateSystemRoleBinding) + systemRoleBindings.POST("/search", SearchSystemRoleBinding) + systemRoleBindings.POST("/update", UpdateSystemRoleBindings) + systemRoleBindings.GET("", ListSystemRoleBindings) + systemRoleBindings.DELETE("/:name", DeleteSystemRoleBinding) + systemRoleBindings.PUT("/:name", CreateOrUpdateSystemRoleBinding) + } + + userBindings := router.Group("userbindings") + { + userBindings.GET("", ListUserBindings) } bundles := router.Group("bundles") { bundles.GET("/:name", DownloadBundle) } + + policyDefinitions := router.Group("policy-definitions") + { + policyDefinitions.GET("", GetPolicyRegistrationDefinitions) + } + + policySvrHealthz := router.Group("healthz") + { + policySvrHealthz.GET("", Healthz) + } + policyUserPermission := router.Group("permission") + { + policyUserPermission.GET("project/:name", GetUserRulesByProject) + policyUserPermission.GET("", GetUserRules) + + } } diff --git a/pkg/microservice/policy/core/repository/models/common.go b/pkg/microservice/policy/core/repository/models/common.go new file mode 100644 index 0000000000000000000000000000000000000000..72881dba58cd01a85ca1aafe2ddf96356c13e8f3 --- /dev/null +++ b/pkg/microservice/policy/core/repository/models/common.go @@ -0,0 +1,52 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +type SubjectKind string + +const ( + MethodAll = "*" + KindResource = "resource" + UserKind SubjectKind = "user" + GroupKind SubjectKind = "group" +) + +// Rule holds information that describes a policy rule, but does not contain information +// about whom the rule applies to. +// If Kind is "resource", verbs are resource actions, while resources are resource names +type Rule struct { + // Verbs is a list of http methods or resource actions that apply to ALL the Resources contained in this rule. '*' represents all methods. + Verbs []string `bson:"verbs" json:"verbs"` + + // Resources is a list of resources this rule applies to. '*' represents all resources. + Resources []string `bson:"resources" json:"resources"` + Kind string `bson:"kind" json:"kind"` + MatchAttributes []MatchAttribute `bson:"match_attributes" json:"match_attributes"` +} + +type MatchAttribute struct { + Key string `json:"key"` + Value string `json:"value"` +} + +// Subject contains a reference to the object or user identities a role binding applies to. +type Subject struct { + // Kind of object being referenced. allowed values are "User", "Group". + Kind SubjectKind `bson:"kind" json:"kind"` + // unique identifier of the object being referenced. + UID string `bson:"uid" json:"uid"` +} diff --git a/pkg/microservice/policy/core/repository/models/policy.go b/pkg/microservice/policy/core/repository/models/policy.go new file mode 100644 index 0000000000000000000000000000000000000000..56d221b41773f06364765f94f7a6cef41332eee9 --- /dev/null +++ b/pkg/microservice/policy/core/repository/models/policy.go @@ -0,0 +1,39 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/koderover/zadig/pkg/setting" +) + +// Policy is a namespaced or cluster scoped, logical grouping of PolicyRules that can be referenced as a unit by a PolicyBinding. +// for a cluster scoped Policy, namespace is empty. +type Policy struct { + Name string `bson:"name" json:"name"` + Namespace string `bson:"namespace" json:"namespace"` + Description string `bson:"description" json:"description"` + CreateTime int64 `bson:"create_time" json:"create_time"` + UpdateTime int64 `bson:"update_time" json:"update_time"` + Rules []*Rule `bson:"rules" json:"rules"` + CreateBy string `bson:"create_by" json:"create_by"` + UpdateBy string `bson:"update_by" json:"update_by"` + Type setting.ResourceType `bson:"type" json:"type"` +} + +func (Policy) TableName() string { + return "policy" +} diff --git a/pkg/microservice/policy/core/repository/models/policy_binding.go b/pkg/microservice/policy/core/repository/models/policy_binding.go new file mode 100644 index 0000000000000000000000000000000000000000..d95a77960328c134c04d2f7e9acc810921947809 --- /dev/null +++ b/pkg/microservice/policy/core/repository/models/policy_binding.go @@ -0,0 +1,48 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/koderover/zadig/pkg/setting" +) + +// PolicyBinding references a Policy, but does not contain it. It adds who information via Subjects. +// PolicyBinding in a given namespace only have effect in that namespace. +// for a cluster scoped PolicyBinding, namespace is empty. +type PolicyBinding struct { + Name string `bson:"name" json:"name"` + Namespace string `bson:"namespace" json:"namespace"` + + // Subjects holds references to the objects the Policy applies to. + Subjects []*Subject `bson:"subjects" json:"subjects"` + + // PolicyRef can reference a namespaced or cluster scoped Policy. + PolicyRef *PolicyRef `bson:"policy_ref" json:"policy_ref"` + Type setting.ResourceType `bson:"type" json:"type"` +} + +// PolicyRef contains information that points to the policy being used +type PolicyRef struct { + Name string `bson:"name" json:"name"` + + // Namespace of the referenced object. if the object is cluster scoped, namespace is empty. + Namespace string `bson:"namespace" json:"namespace"` +} + +func (PolicyBinding) TableName() string { + return "policybinding" +} diff --git a/pkg/microservice/policy/core/repository/models/policy_meta.go b/pkg/microservice/policy/core/repository/models/policy_meta.go new file mode 100644 index 0000000000000000000000000000000000000000..eaaecb7dcb39cb40dad81e8d9929ca919061045a --- /dev/null +++ b/pkg/microservice/policy/core/repository/models/policy_meta.go @@ -0,0 +1,48 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +type PolicyMeta struct { + Resource string `bson:"resource" json:"resource"` + Alias string `bson:"alias" json:"alias"` + Description string `bson:"description" json:"description"` + Rules []*PolicyMetaRule `bson:"rules" json:"rules"` +} + +type PolicyMetaRule struct { + Action string `bson:"action" json:"action"` + Alias string `bson:"alias" json:"alias"` + Description string `bson:"description" json:"description"` + Rules []*ActionRule `bson:"rules" json:"rules"` +} + +type ActionRule struct { + Method string `bson:"method" json:"method"` + Endpoint string `bson:"endpoint" json:"endpoint"` + ResourceType string `bson:"resource_type,omitempty" json:"resource_type,omitempty"` + IDRegex string `bson:"id_regex,omitempty" json:"idRegex,omitempty"` + MatchAttributes []Attribute `bson:"match_attributes,omitempty" json:"match_attributes,omitempty"` +} + +type Attribute struct { + Key string `bson:"key" json:"key"` + Value string `bson:"value" json:"value"` +} + +func (PolicyMeta) TableName() string { + return "policy_meta" +} diff --git a/pkg/microservice/policy/core/repository/models/role.go b/pkg/microservice/policy/core/repository/models/role.go index a49233a163fb9fa8a2acb4abdc672d81743610a9..f2a65f5395c0e99f5732a36e171be1d97e0b1bf9 100644 --- a/pkg/microservice/policy/core/repository/models/role.go +++ b/pkg/microservice/policy/core/repository/models/role.go @@ -16,31 +16,20 @@ limitations under the License. package models -const ( - MethodAll = "*" - KindResource = "resource" +import ( + "github.com/koderover/zadig/pkg/setting" ) // Role is a namespaced or cluster scoped, logical grouping of PolicyRules that can be referenced as a unit by a RoleBinding. // for a cluster scoped Role, namespace is empty. type Role struct { - Name string `bson:"name" json:"name"` - Namespace string `bson:"namespace" json:"namespace"` - Rules []*PolicyRule `bson:"rules" json:"rules"` - Kind string `bson:"kind" json:"kind"` -} - -// PolicyRule holds information that describes a policy rule, but does not contain information -// about whom the rule applies to. -// If Kind is "resource", verbs are resource actions, while resources are resource names -type PolicyRule struct { - // Verbs is a list of http methods or resource actions that apply to ALL the Resources contained in this rule. '*' represents all methods. - Verbs []string `bson:"verbs" json:"verbs"` - - // Resources is a list of resources this rule applies to. '*' represents all resources. - Resources []string `bson:"resources" json:"resources"` + Name string `bson:"name" json:"name"` + Desc string `bson:"desc" json:"desc"` + Namespace string `bson:"namespace" json:"namespace"` + Rules []*Rule `bson:"rules" json:"rules"` + Type setting.ResourceType `bson:"type" json:"type"` } func (Role) TableName() string { - return "policy_role" + return "role" } diff --git a/pkg/microservice/policy/core/repository/models/role_binding.go b/pkg/microservice/policy/core/repository/models/role_binding.go index 956b1b1b7f3bbf4d4c59dee56e243ace7055318f..fa75f046e6b92d6f731c3790390bd18bcdafed3d 100644 --- a/pkg/microservice/policy/core/repository/models/role_binding.go +++ b/pkg/microservice/policy/core/repository/models/role_binding.go @@ -16,13 +16,6 @@ limitations under the License. package models -type SubjectKind string - -const ( - UserKind SubjectKind = "user" - GroupKind SubjectKind = "group" -) - // RoleBinding references a role, but does not contain it. It adds who information via Subjects. // RoleBindings in a given namespace only have effect in that namespace. // for a cluster scoped RoleBinding, namespace is empty. @@ -37,14 +30,6 @@ type RoleBinding struct { RoleRef *RoleRef `bson:"role_ref" json:"roleRef"` } -// Subject contains a reference to the object or user identities a role binding applies to. -type Subject struct { - // Kind of object being referenced. allowed values are "User", "Group". - Kind SubjectKind `bson:"kind" json:"kind"` - // Name of the object being referenced. - Name string `bson:"name" json:"name"` -} - // RoleRef contains information that points to the role being used type RoleRef struct { Name string `bson:"name" json:"name"` diff --git a/pkg/microservice/policy/core/repository/mongodb/policy.go b/pkg/microservice/policy/core/repository/mongodb/policy.go new file mode 100644 index 0000000000000000000000000000000000000000..88b8919f71c96b7731ce87363d27d6b14f68059f --- /dev/null +++ b/pkg/microservice/policy/core/repository/mongodb/policy.go @@ -0,0 +1,217 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "errors" + "fmt" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/policy/core/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type PolicyColl struct { + *mongo.Collection + + coll string +} + +func NewPolicyColl() *PolicyColl { + name := models.Policy{}.TableName() + return &PolicyColl{ + Collection: mongotool.Database(config.MongoDatabase()).Collection(name), + coll: name, + } +} + +func (c *PolicyColl) GetCollectionName() string { + return c.coll +} + +func (c *PolicyColl) EnsureIndex(ctx context.Context) error { + mod := mongo.IndexModel{ + Keys: bson.D{ + bson.E{Key: "namespace", Value: 1}, + bson.E{Key: "name", Value: 1}, + }, + Options: options.Index().SetUnique(true), + } + + _, err := c.Indexes().CreateOne(ctx, mod) + + return err +} + +func (c *PolicyColl) GetByNames(names []string) ([]*models.Policy, error) { + query := bson.M{} + query["name"] = bson.M{"$in": names} + + cursor, err := c.Collection.Find(context.TODO(), query) + if err != nil { + return nil, err + } + + var res []*models.Policy + err = cursor.All(context.TODO(), &res) + if err != nil { + return nil, err + } + return res, nil +} + +func (c *PolicyColl) Get(ns, name string) (*models.Policy, bool, error) { + res := &models.Policy{} + + query := bson.M{"namespace": ns, "name": name} + err := c.FindOne(context.TODO(), query).Decode(res) + if err != nil { + if err == mongo.ErrNoDocuments { + return nil, false, nil + } + + return nil, false, err + } + + return res, true, nil +} + +func (c *PolicyColl) List() ([]*models.Policy, error) { + var res []*models.Policy + + ctx := context.Background() + + cursor, err := c.Collection.Find(ctx, bson.M{}) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &res) + if err != nil { + return nil, err + } + return res, nil +} + +func (c *PolicyColl) ListBy(projectName string) ([]*models.Policy, error) { + var res []*models.Policy + + ctx := context.Background() + query := bson.M{"namespace": projectName} + + cursor, err := c.Collection.Find(ctx, query) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &res) + if err != nil { + return nil, err + } + return res, nil +} + +func (c *PolicyColl) ListBySpaceAndName(projectName string, name string) ([]*models.Policy, error) { + var res []*models.Policy + + ctx := context.Background() + query := bson.M{"namespace": projectName, "name": name} + + cursor, err := c.Collection.Find(ctx, query) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &res) + if err != nil { + return nil, err + } + return res, nil +} + +func (c *PolicyColl) Create(obj *models.Policy) error { + if obj == nil { + return fmt.Errorf("nil object") + } + + _, err := c.InsertOne(context.TODO(), obj) + + return err +} + +func (c *PolicyColl) BulkCreate(args []*models.Policy) error { + if len(args) == 0 { + return nil + } + var ois []interface{} + for _, obj := range args { + ois = append(ois, obj) + } + _, err := c.InsertMany(context.TODO(), ois) + return err +} + +func (c *PolicyColl) Delete(name string, projectName string) error { + query := bson.M{"name": name, "namespace": projectName} + _, err := c.DeleteOne(context.TODO(), query) + return err +} + +func (c *PolicyColl) DeleteMany(names []string, projectName string) error { + query := bson.M{} + if len(projectName) > 0 { + query = bson.M{"namespace": projectName} + } + if len(names) > 0 { + query["name"] = bson.M{"$in": names} + } + _, err := c.Collection.DeleteMany(context.TODO(), query) + return err +} + +func (c *PolicyColl) UpdatePolicy(obj *models.Policy) error { + // avoid panic issue + if obj == nil { + return errors.New("nil Policy") + } + + query := bson.M{"name": obj.Name, "namespace": obj.Namespace} + change := bson.M{"$set": bson.M{ + "rules": obj.Rules, + "description": obj.Description, + "update_time": obj.UpdateTime, + }} + _, err := c.UpdateOne(context.TODO(), query, change) + return err +} + +func (c *PolicyColl) UpdateOrCreate(obj *models.Policy) error { + if obj == nil { + return fmt.Errorf("nil object") + } + + query := bson.M{"name": obj.Name, "namespace": obj.Namespace} + opts := options.Replace().SetUpsert(true) + _, err := c.ReplaceOne(context.TODO(), query, obj, opts) + + return err +} diff --git a/pkg/microservice/policy/core/repository/mongodb/policy_binding.go b/pkg/microservice/policy/core/repository/mongodb/policy_binding.go new file mode 100644 index 0000000000000000000000000000000000000000..26575501d847fec7e40cf1febf2be32432606f3b --- /dev/null +++ b/pkg/microservice/policy/core/repository/mongodb/policy_binding.go @@ -0,0 +1,205 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "fmt" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/policy/core/repository/models" + "github.com/koderover/zadig/pkg/tool/log" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type ListPolicyOptions struct { + PolicyName, PolicyNamespace string +} + +type PolicyBindingColl struct { + *mongo.Collection + + coll string +} + +func NewPolicyBindingColl() *PolicyBindingColl { + name := models.PolicyBinding{}.TableName() + return &PolicyBindingColl{ + Collection: mongotool.Database(config.MongoDatabase()).Collection(name), + coll: name, + } +} + +func (c *PolicyBindingColl) GetCollectionName() string { + return c.coll +} + +func (c *PolicyBindingColl) EnsureIndex(ctx context.Context) error { + mod := mongo.IndexModel{ + Keys: bson.D{ + bson.E{Key: "namespace", Value: 1}, + bson.E{Key: "name", Value: 1}, + }, + Options: options.Index().SetUnique(true), + } + + _, err := c.Indexes().CreateOne(ctx, mod) + + return err +} + +func (c *PolicyBindingColl) List(opts ...*ListPolicyOptions) ([]*models.PolicyBinding, error) { + var res []*models.PolicyBinding + + ctx := context.Background() + query := bson.M{} + if len(opts) > 0 { + opt := opts[0] + if opt.PolicyName != "" { + query["policy_ref.name"] = opt.PolicyName + query["policy_ref.namespace"] = opt.PolicyNamespace + } + } + + cursor, err := c.Collection.Find(ctx, query) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &res) + if err != nil { + return nil, err + } + + return res, nil +} + +func (c *PolicyBindingColl) ListBy(projectName, uid string) ([]*models.PolicyBinding, error) { + var res []*models.PolicyBinding + + ctx := context.Background() + query := bson.M{"namespace": projectName} + if uid != "" { + query["subjects.uid"] = uid + query["subjects.kind"] = models.UserKind + } + + cursor, err := c.Collection.Find(ctx, query) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &res) + if err != nil { + return nil, err + } + + return res, nil +} + +func (c *PolicyBindingColl) Delete(name string, projectName string) error { + query := bson.M{"name": name, "namespace": projectName} + _, err := c.DeleteOne(context.TODO(), query) + return err +} + +func (c *PolicyBindingColl) DeleteMany(names []string, projectName string, userID string) error { + query := bson.M{} + if projectName != "" { + query["namespace"] = projectName + } + if len(names) > 0 { + query["name"] = bson.M{"$in": names} + } + + if userID != "" { + query = bson.M{"subjects.uid": userID} + } + _, err := c.Collection.DeleteMany(context.TODO(), query) + + return err +} + +func (c *PolicyBindingColl) DeleteByPolicy(policyName string, projectName string) error { + query := bson.M{"policy_ref.name": policyName, "policy_ref.namespace": projectName} + // if projectName == "", delete all policybindings in all namespaces + if projectName != "" { + query["namespace"] = projectName + } + _, err := c.Collection.DeleteMany(context.TODO(), query) + + return err +} + +func (c *PolicyBindingColl) DeleteByPolicies(policyNames []string, projectName string) error { + query := bson.M{} + if len(projectName) != 0 { + query = bson.M{"policy_ref.namespace": projectName, "namespace": projectName} + } + + if len(policyNames) != 0 { + query["policy_ref.name"] = bson.M{"$in": policyNames} + } + _, err := c.Collection.DeleteMany(context.TODO(), query) + + return err +} + +func (c *PolicyBindingColl) Create(obj *models.PolicyBinding) error { + if obj == nil { + return fmt.Errorf("nil object") + } + + _, err := c.InsertOne(context.TODO(), obj) + + return err +} + +func (c *PolicyBindingColl) BulkCreate(objs []*models.PolicyBinding) error { + if len(objs) == 0 { + return nil + } + + var ois []interface{} + for _, obj := range objs { + ois = append(ois, obj) + } + + res, err := c.InsertMany(context.TODO(), ois) + if mongo.IsDuplicateKeyError(err) { + log.Warnf("Duplicate key found, inserted IDs is %v", res.InsertedIDs) + return nil + } + + return err +} + +func (c *PolicyBindingColl) UpdateOrCreate(obj *models.PolicyBinding) error { + if obj == nil { + return fmt.Errorf("nil object") + } + + query := bson.M{"name": obj.Name, "namespace": obj.Namespace} + opts := options.Replace().SetUpsert(true) + _, err := c.ReplaceOne(context.TODO(), query, obj, opts) + + return err +} diff --git a/pkg/microservice/policy/core/repository/mongodb/policy_meta_registration.go b/pkg/microservice/policy/core/repository/mongodb/policy_meta_registration.go new file mode 100644 index 0000000000000000000000000000000000000000..a1c14664c5daac71e293a4311a7a95167aef6dac --- /dev/null +++ b/pkg/microservice/policy/core/repository/mongodb/policy_meta_registration.go @@ -0,0 +1,88 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "fmt" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/policy/core/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type PolicyMetaColl struct { + *mongo.Collection + + coll string +} + +func NewPolicyMetaColl() *PolicyMetaColl { + name := models.PolicyMeta{}.TableName() + return &PolicyMetaColl{ + Collection: mongotool.Database(config.MongoDatabase()).Collection(name), + coll: name, + } +} + +func (c *PolicyMetaColl) GetCollectionName() string { + return c.coll +} + +func (c *PolicyMetaColl) EnsureIndex(ctx context.Context) error { + mod := mongo.IndexModel{ + Keys: bson.M{"resource": 1}, + Options: options.Index().SetUnique(true), + } + + _, err := c.Indexes().CreateOne(ctx, mod) + + return err +} + +func (c *PolicyMetaColl) List() ([]*models.PolicyMeta, error) { + var res []*models.PolicyMeta + + ctx := context.Background() + cursor, err := c.Collection.Find(ctx, bson.M{}) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &res) + if err != nil { + return nil, err + } + + return res, nil +} + +func (c *PolicyMetaColl) UpdateOrCreate(obj *models.PolicyMeta) error { + if obj == nil { + return fmt.Errorf("nil object") + } + + query := bson.M{"resource": obj.Resource} + opts := options.Replace().SetUpsert(true) + _, err := c.ReplaceOne(context.TODO(), query, obj, opts) + + return err +} diff --git a/pkg/microservice/policy/core/repository/mongodb/role.go b/pkg/microservice/policy/core/repository/mongodb/role.go index da8595c9ca1b659413de613bb364264558ff55b6..c17986acd63909eecfc6f24d6110cd30fb1c727f 100644 --- a/pkg/microservice/policy/core/repository/mongodb/role.go +++ b/pkg/microservice/policy/core/repository/mongodb/role.go @@ -18,6 +18,7 @@ package mongodb import ( "context" + "errors" "fmt" "go.mongodb.org/mongo-driver/bson" @@ -81,6 +82,7 @@ func (c *RoleColl) List() ([]*models.Role, error) { var res []*models.Role ctx := context.Background() + cursor, err := c.Collection.Find(ctx, bson.M{}) if err != nil { return nil, err @@ -90,7 +92,42 @@ func (c *RoleColl) List() ([]*models.Role, error) { if err != nil { return nil, err } + return res, nil +} + +func (c *RoleColl) ListBy(projectName string) ([]*models.Role, error) { + var res []*models.Role + + ctx := context.Background() + query := bson.M{"namespace": projectName} + cursor, err := c.Collection.Find(ctx, query) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &res) + if err != nil { + return nil, err + } + return res, nil +} + +func (c *RoleColl) ListBySpaceAndName(projectName string, name string) ([]*models.Role, error) { + var res []*models.Role + + ctx := context.Background() + query := bson.M{"namespace": projectName, "name": name} + + cursor, err := c.Collection.Find(ctx, query) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &res) + if err != nil { + return nil, err + } return res, nil } @@ -103,3 +140,44 @@ func (c *RoleColl) Create(obj *models.Role) error { return err } + +func (c *RoleColl) Delete(name string, projectName string) error { + query := bson.M{"name": name, "namespace": projectName} + _, err := c.DeleteOne(context.TODO(), query) + return err +} + +func (c *RoleColl) DeleteMany(names []string, projectName string) error { + query := bson.M{"namespace": projectName} + if len(names) > 0 { + query["name"] = bson.M{"$in": names} + } + _, err := c.Collection.DeleteMany(context.TODO(), query) + return err +} + +func (c *RoleColl) UpdateRole(obj *models.Role) error { + // avoid panic issue + if obj == nil { + return errors.New("nil Role") + } + + query := bson.M{"name": obj.Name, "namespace": obj.Namespace} + change := bson.M{"$set": bson.M{ + "rules": obj.Rules, + }} + _, err := c.UpdateOne(context.TODO(), query, change) + return err +} + +func (c *RoleColl) UpdateOrCreate(obj *models.Role) error { + if obj == nil { + return fmt.Errorf("nil object") + } + + query := bson.M{"name": obj.Name, "namespace": obj.Namespace} + opts := options.Replace().SetUpsert(true) + _, err := c.ReplaceOne(context.TODO(), query, obj, opts) + + return err +} diff --git a/pkg/microservice/policy/core/repository/mongodb/role_binding.go b/pkg/microservice/policy/core/repository/mongodb/role_binding.go index 570d53f6b338fe2a0da461be1300f40af78c5517..ed86f765c8250f3e91af632c91415f5d06aef3b1 100644 --- a/pkg/microservice/policy/core/repository/mongodb/role_binding.go +++ b/pkg/microservice/policy/core/repository/mongodb/role_binding.go @@ -26,9 +26,14 @@ import ( "github.com/koderover/zadig/pkg/config" "github.com/koderover/zadig/pkg/microservice/policy/core/repository/models" + "github.com/koderover/zadig/pkg/tool/log" mongotool "github.com/koderover/zadig/pkg/tool/mongo" ) +type ListOptions struct { + RoleName, RoleNamespace string +} + type RoleBindingColl struct { *mongo.Collection @@ -61,11 +66,87 @@ func (c *RoleBindingColl) EnsureIndex(ctx context.Context) error { return err } -func (c *RoleBindingColl) List() ([]*models.RoleBinding, error) { +func (c *RoleBindingColl) List(opts ...*ListOptions) ([]*models.RoleBinding, error) { + var res []*models.RoleBinding + + ctx := context.Background() + query := bson.M{} + if len(opts) > 0 { + opt := opts[0] + if opt.RoleName != "" { + query["role_ref.name"] = opt.RoleName + query["role_ref.namespace"] = opt.RoleNamespace + } + } + + cursor, err := c.Collection.Find(ctx, query) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &res) + if err != nil { + return nil, err + } + + return res, nil +} + +func (c *RoleBindingColl) ListBy(projectName, uid string) ([]*models.RoleBinding, error) { + var res []*models.RoleBinding + + ctx := context.Background() + query := bson.M{"namespace": projectName} + if uid != "" { + query["subjects.uid"] = uid + query["subjects.kind"] = models.UserKind + } + + cursor, err := c.Collection.Find(ctx, query) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &res) + if err != nil { + return nil, err + } + + return res, nil +} + +func (c *RoleBindingColl) ListRoleBindingsByUIDs(uids []string) ([]*models.RoleBinding, error) { + var res []*models.RoleBinding + + ctx := context.Background() + query := bson.M{} + if len(uids) > 0 { + query["subjects.uid"] = bson.M{"$in": uids} + } + + cursor, err := c.Collection.Find(ctx, query) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &res) + if err != nil { + return nil, err + } + + return res, nil +} + +func (c *RoleBindingColl) ListSystemRoleBindingsByUIDs(uids []string) ([]*models.RoleBinding, error) { var res []*models.RoleBinding ctx := context.Background() - cursor, err := c.Collection.Find(ctx, bson.M{}) + query := bson.M{"namespace": "*"} + if len(uids) > 0 { + query["subjects.uid"] = bson.M{"$in": uids} + } + + cursor, err := c.Collection.Find(ctx, query) if err != nil { return nil, err } @@ -78,6 +159,54 @@ func (c *RoleBindingColl) List() ([]*models.RoleBinding, error) { return res, nil } +func (c *RoleBindingColl) Delete(name string, projectName string) error { + query := bson.M{"name": name, "namespace": projectName} + _, err := c.DeleteOne(context.TODO(), query) + return err +} + +func (c *RoleBindingColl) DeleteMany(names []string, projectName string, userID string) error { + query := bson.M{} + if projectName != "" { + query["namespace"] = projectName + } + if len(names) > 0 { + query["name"] = bson.M{"$in": names} + } + + if userID != "" { + query["subjects.uid"] = userID + } + _, err := c.Collection.DeleteMany(context.TODO(), query) + + return err +} + +func (c *RoleBindingColl) DeleteByRole(roleName string, projectName string) error { + query := bson.M{"role_ref.name": roleName, "role_ref.namespace": projectName} + // if projectName == "", delete all rolebindings in all namespaces + if projectName != "" { + query["namespace"] = projectName + } + _, err := c.Collection.DeleteMany(context.TODO(), query) + + return err +} + +func (c *RoleBindingColl) DeleteByRoles(roleNames []string, projectName string) error { + if projectName == "" { + return fmt.Errorf("projectName is empty") + } + if len(roleNames) == 0 { + return nil + } + + query := bson.M{"role_ref.name": bson.M{"$in": roleNames}, "role_ref.namespace": projectName, "namespace": projectName} + _, err := c.Collection.DeleteMany(context.TODO(), query) + + return err +} + func (c *RoleBindingColl) Create(obj *models.RoleBinding) error { if obj == nil { return fmt.Errorf("nil object") @@ -87,3 +216,70 @@ func (c *RoleBindingColl) Create(obj *models.RoleBinding) error { return err } + +func (c *RoleBindingColl) BulkCreate(objs []*models.RoleBinding) error { + if len(objs) == 0 { + return nil + } + + var ois []interface{} + for _, obj := range objs { + ois = append(ois, obj) + } + + res, err := c.InsertMany(context.TODO(), ois) + if mongo.IsDuplicateKeyError(err) { + log.Warnf("Duplicate key found, inserted IDs is %v", res.InsertedIDs) + return nil + } + + return err +} + +func (c *RoleBindingColl) UpdateOrCreate(obj *models.RoleBinding) error { + if obj == nil { + return fmt.Errorf("nil object") + } + + query := bson.M{"name": obj.Name, "namespace": obj.Namespace} + opts := options.Replace().SetUpsert(true) + _, err := c.ReplaceOne(context.TODO(), query, obj, opts) + + return err +} + +type RoleBinding struct { + Uid string `json:"uid"` + Namespace string `json:"namespace"` +} + +type ListRoleBindingsOpt struct { + RoleBindings []RoleBinding +} + +func (c *RoleBindingColl) ListByRoleBindingOpt(opt ListRoleBindingsOpt) ([]*models.RoleBinding, error) { + var res []*models.RoleBinding + + if len(opt.RoleBindings) == 0 { + return nil, nil + } + condition := bson.A{} + for _, meta := range opt.RoleBindings { + condition = append(condition, bson.M{ + "namespace": meta.Namespace, + "subjects.uid": meta.Uid, + }) + } + filter := bson.D{{"$or", condition}} + cursor, err := c.Collection.Find(context.TODO(), filter) + if err == mongo.ErrNoDocuments { + return nil, nil + } + if err != nil { + return nil, err + } + if err := cursor.All(context.TODO(), &res); err != nil { + return nil, err + } + return res, nil +} diff --git a/pkg/microservice/policy/core/service.go b/pkg/microservice/policy/core/service.go index 4fc79dce8435eb915386b38dc6824997784caad7..ff16117c662b04292f3d837889b413e2fc12b633 100644 --- a/pkg/microservice/policy/core/service.go +++ b/pkg/microservice/policy/core/service.go @@ -67,7 +67,7 @@ func Start(ctx context.Context) { go StartControllers(ctx.Done()) - bundle.RefreshOPABundle() + bundle.GenerateOPABundle() } func Stop(ctx context.Context) { @@ -90,6 +90,7 @@ func initDatabase(ctx context.Context) { for _, r := range []indexer{ mongodb.NewRoleColl(), mongodb.NewRoleBindingColl(), + mongodb.NewPolicyMetaColl(), } { wg.Add(1) go func(r indexer) { @@ -107,3 +108,7 @@ type indexer interface { EnsureIndex(ctx context.Context) error GetCollectionName() string } + +func Healthz(ctx context.Context) error { + return mongotool.Ping(ctx) +} diff --git a/pkg/microservice/policy/core/service/bundle/exemption_urls.go b/pkg/microservice/policy/core/service/bundle/exemption_urls.go deleted file mode 100644 index 62d041a4892f3d1bfa3c710257168cd0753b5b3f..0000000000000000000000000000000000000000 --- a/pkg/microservice/policy/core/service/bundle/exemption_urls.go +++ /dev/null @@ -1,45 +0,0 @@ -package bundle - -type exemptionURLs struct { - Global rules `json:"global"` // global urls are only controlled by AuthN, and it is visible for all users - Namespaced rules `json:"namespaced"` // global urls are only controlled by AuthN, and it is visible for users under certain projects - Public rules `json:"public"` // public urls are not controlled by AuthN and AuthZ -} - -type policyRule struct { - Methods []string `json:"methods"` - Endpoints []string `json:"endpoints"` -} - -var globalURLs = []*policyRule{ - { - Methods: []string{"*"}, - Endpoints: []string{"api/v1/version"}, - }, -} - -var namespacedURLs = []*policyRule{ - { - Methods: []string{"*"}, - Endpoints: []string{"api/v1/version1"}, - }, -} - -var publicURLs = []*policyRule{ - { - Methods: []string{"GET"}, - Endpoints: []string{"api/aslan/health"}, - }, - { - Methods: []string{"POST"}, - Endpoints: []string{"api/aslan/webhook"}, - }, - { - Methods: []string{"GET"}, - Endpoints: []string{"api/hub/connect"}, - }, - { - Methods: []string{"GET"}, - Endpoints: []string{"api/aslan/kodespace/downloadUrl"}, - }, -} diff --git a/pkg/microservice/policy/core/service/bundle/opa_bundle.go b/pkg/microservice/policy/core/service/bundle/opa_bundle.go index ca22f71f9e9949a30530fc9fc510ebfedca30186..9e6107df2a2f63da0b220f48e2f11aa09dff940f 100644 --- a/pkg/microservice/policy/core/service/bundle/opa_bundle.go +++ b/pkg/microservice/policy/core/service/bundle/opa_bundle.go @@ -17,137 +17,172 @@ limitations under the License. package bundle import ( - "encoding/json" "net/http" - "path/filepath" "sort" - "github.com/27149chen/afero" - "github.com/google/uuid" + "k8s.io/apimachinery/pkg/util/sets" "github.com/koderover/zadig/pkg/config" "github.com/koderover/zadig/pkg/microservice/policy/core/repository/models" "github.com/koderover/zadig/pkg/microservice/policy/core/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/policy/core/yamlconfig" "github.com/koderover/zadig/pkg/tool/log" - fsutil "github.com/koderover/zadig/pkg/util/fs" + "github.com/koderover/zadig/pkg/tool/opa" ) const ( - manifestPath = ".manifest" - policyPath = "authz.rego" - rolesPath = "roles/data.json" - rolebindingsPath = "bindings/data.json" - exemptionPath = "exemptions/data.json" + manifestPath = ".manifest" + policyRegoPath = "authz.rego" + rolesPath = "roles/data.json" + policiesPath = "policies/data.json" + bindingsPath = "bindings/data.json" + + exemptionsPath = "exemptions/data.json" + resourcesPath = "resources/data.json" + + policyRoot = "rbac" + rolesRoot = "roles" + rolebindingsRoot = "bindings" + exemptionsRoot = "exemptions" + resourcesRoot = "resources" + policiesRoot = "policies" ) +type expressionOperator string + const ( - MethodView = "VIEW" - MethodGet = http.MethodGet - MethodPost = http.MethodPost - MethodPut = http.MethodPut - MethodPatch = http.MethodPatch - MethodDelete = http.MethodDelete - ActionCreate = "create" - ActionDelete = "delete" - ActionDeleteCollection = "deletecollection" - ActionGet = "get" - ActionList = "list" - ActionPatch = "patch" - ActionUpdate = "update" - ActionWatch = "watch" + OperatorIn expressionOperator = "In" + OperatorNotIn expressionOperator = "NotIn" + OperatorExists expressionOperator = "Exists" + OperatorDoesNotExist expressionOperator = "DoesNotExist" +) + +const ( + MethodGet = http.MethodGet + MethodPost = http.MethodPost + MethodPut = http.MethodPut + MethodPatch = http.MethodPatch + MethodDelete = http.MethodDelete ) var AllMethods = []string{MethodGet, MethodPost, MethodPut, MethodPatch, MethodDelete} -var AllActions = []string{ActionCreate, ActionDelete, ActionDeleteCollection, ActionGet, ActionList, ActionPatch, ActionUpdate, ActionWatch} -var cacheFS afero.Fs +var revision string type opaRoles struct { Roles roles `json:"roles"` } -type opaRoleBindings struct { - RoleBindings roleBindings `json:"role_bindings"` +type opaPolicies struct { + Roles roles `json:"policies"` } -type opaManifest struct { - Revision string `json:"revision"` - Roots []string `json:"roots"` +type opaRoleBindings struct { + RoleBindings roleBindings `json:"role_bindings"` + PolicyBindings policyBindings `json:"policy_bindings"` } type role struct { Name string `json:"name"` Namespace string `json:"namespace"` - Rules rules `json:"rules"` + Rules Rules `json:"rules"` } -type rule struct { - Method string `json:"method"` - Endpoint string `json:"endpoint"` +type Rule struct { + Method string `json:"method"` + Endpoint string `json:"endpoint"` + ResourceType string `json:"resourceType,omitempty"` + IDRegex string `json:"idRegex,omitempty"` + MatchAttributes Attributes `json:"matchAttributes,omitempty"` + MatchExpressions []expression `json:"matchExpressions,omitempty"` } -type roleBinding struct { - User string `json:"user"` - RoleRefs roleRefs `json:"role_refs"` +type Attribute struct { + Key string `json:"key"` + Value string `json:"value"` } -type roleRef struct { - Name string `json:"name"` - Namespace string `json:"namespace"` -} - -type opaDataSpec struct { - data interface{} - path string +func NewFrom(attrs []models.MatchAttribute) Attributes { + var att Attributes + for _, v := range attrs { + a := Attribute{ + Key: v.Key, + Value: v.Value, + } + att = append(att, &a) + } + return att } -type opaData []*opaDataSpec - -func (o *opaData) save() error { - var err error - - cacheFS = afero.NewMemMapFs() - for _, file := range *o { - var content []byte - switch c := file.data.(type) { - case []byte: - content = c - default: - content, err = json.MarshalIndent(c, "", " ") - if err != nil { - log.Errorf("Failed to marshal file %s, err: %s", file.path, err) - return err - } - } +type Attributes []*Attribute - err = cacheFS.MkdirAll(filepath.Dir(file.path), 0755) - if err != nil { - log.Errorf("Failed to create path %s, err: %s", filepath.Dir(file.path), err) - return err +func (a Attributes) LessOrEqual(other Attributes) bool { + if len(a) == 0 { + return true + } + if len(other) == 0 { + return false + } + for i, attr := range a { + if i > len(other)-1 { + return false } - err = afero.WriteFile(cacheFS, file.path, content, 0644) - if err != nil { - log.Errorf("Failed to write file %s, err: %s", file.path, err) - return err + ki := attr.Key + kj := other[i].Key + vi := attr.Value + vj := other[i].Value + if ki == kj { + if vi == vj { + continue + } + return vi < vj } + return ki < kj } - tarball := "bundle.tar.gz" - path := filepath.Join(config.DataPath(), tarball) - if err = fsutil.Tar(afero.NewIOFS(cacheFS), path); err != nil { - log.Errorf("Failed to archive tarball %s, err: %s", path, err) - return err - } + return true +} + +type expression struct { + Key string `json:"key"` + Values []string `json:"values"` + Operator expressionOperator `json:"operator"` +} + +type roleBinding struct { + UID string `json:"uid"` + Bindings bindings `json:"bindings"` +} + +type policyBinding struct { + UID string `json:"uid"` + Bindings bindingPolicys `json:"bindings"` +} + +type binding struct { + Namespace string `json:"namespace"` + RoleRefs roleRefs `json:"role_refs"` +} + +type bindingPolicy struct { + Namespace string `json:"namespace"` + RoleRefs roleRefs `json:"policy_refs"` +} - return nil +type roleRef struct { + Name string `json:"name"` + Namespace string `json:"namespace"` } -type rules []*rule +type Rules []*Rule -func (o rules) Len() int { return len(o) } -func (o rules) Swap(i, j int) { o[i], o[j] = o[j], o[i] } -func (o rules) Less(i, j int) bool { +func (o Rules) Len() int { return len(o) } +func (o Rules) Swap(i, j int) { o[i], o[j] = o[j], o[i] } +func (o Rules) Less(i, j int) bool { if o[i].Endpoint == o[j].Endpoint { + if o[i].Method == o[j].Method { + return o[i].MatchAttributes.LessOrEqual(o[j].MatchAttributes) + } return o[i].Method < o[j].Method } return o[i].Endpoint < o[j].Endpoint @@ -164,6 +199,22 @@ func (o roles) Less(i, j int) bool { return o[i].Namespace < o[j].Namespace } +type bindings []*binding + +func (o bindings) Len() int { return len(o) } +func (o bindings) Swap(i, j int) { o[i], o[j] = o[j], o[i] } +func (o bindings) Less(i, j int) bool { + return o[i].Namespace < o[j].Namespace +} + +type bindingPolicys []*bindingPolicy + +func (o bindingPolicys) Len() int { return len(o) } +func (o bindingPolicys) Swap(i, j int) { o[i], o[j] = o[j], o[i] } +func (o bindingPolicys) Less(i, j int) bool { + return o[i].Namespace < o[j].Namespace +} + type roleRefs []*roleRef func (o roleRefs) Len() int { return len(o) } @@ -180,196 +231,300 @@ type roleBindings []*roleBinding func (o roleBindings) Len() int { return len(o) } func (o roleBindings) Swap(i, j int) { o[i], o[j] = o[j], o[i] } func (o roleBindings) Less(i, j int) bool { - return o[i].User < o[j].User + return o[i].UID < o[j].UID +} + +type policyBindings []*policyBinding + +func (o policyBindings) Len() int { return len(o) } +func (o policyBindings) Swap(i, j int) { o[i], o[j] = o[j], o[i] } +func (o policyBindings) Less(i, j int) bool { + return o[i].UID < o[j].UID } -func generateOPAResourceRoles(roles []*models.Role) *opaRoles { +func generateOPARoles(roles []*models.Role, policyMetas []*models.PolicyMeta) *opaRoles { data := &opaRoles{} + resourceMappings := getResourceActionMappings(false, policyMetas) for _, ro := range roles { - opaRole := &role{Name: ro.Name, Namespace: ro.Namespace} + verbAttrMap := make(map[string]sets.String) + resourceVerbs := make(map[string]sets.String) for _, r := range ro.Rules { - if len(r.Verbs) == 1 && r.Verbs[0] == models.MethodAll { - r.Verbs = AllActions - } - for _, res := range r.Resources { - if mapping, ok := mappings[res]; !ok { - continue + for _, verb := range r.Verbs { + if verbs, ok := resourceVerbs[r.Resources[0]]; ok { + for _, v := range r.Verbs { + verbs.Insert(v) + } + resourceVerbs[r.Resources[0]] = verbs } else { + verbSet := sets.String{} for _, v := range r.Verbs { - opaRole.Rules = append(opaRole.Rules, mapping[v]...) + verbSet.Insert(v) } + resourceVerbs[r.Resources[0]] = verbSet + } + if attrs, ok := verbAttrMap[verb]; ok { + for _, attribute := range r.MatchAttributes { + attrs.Insert(attribute.Key + "&&" + attribute.Value) + } + verbAttrMap[verb] = attrs + } else { + attrSet := sets.String{} + for _, attribute := range r.MatchAttributes { + attrSet.Insert(attribute.Key + "&&" + attribute.Value) + } + verbAttrMap[verb] = attrSet } - } } + //TODO - mouuii change role model to policy model + opaRole := &role{Name: ro.Name, Namespace: ro.Namespace} + for resource, verbs := range resourceVerbs { + ruleList := resourceMappings.GetPolicyRules(resource, verbs.List(), verbAttrMap) + opaRole.Rules = append(opaRole.Rules, ruleList...) + } + for _, r := range ro.Rules { + if r.Kind != models.KindResource { + if len(r.Verbs) == 1 && r.Verbs[0] == models.MethodAll { + r.Verbs = AllMethods + } + for _, v := range r.Verbs { + for _, endpoint := range r.Resources { + opaRole.Rules = append(opaRole.Rules, &Rule{Method: v, Endpoint: endpoint}) + } + } + } + } sort.Sort(opaRole.Rules) data.Roles = append(data.Roles, opaRole) } - sort.Sort(data.Roles) - return data } -func generateOPARoles(roles []*models.Role) *opaRoles { - data := &opaRoles{} +func generateOPAPolicies(policies []*models.Policy, policyMetas []*models.PolicyMeta) *opaPolicies { + data := &opaPolicies{} + resourceMappings := getResourceActionMappings(true, policyMetas) - for _, ro := range roles { - opaRole := &role{Name: ro.Name, Namespace: ro.Namespace} - if ro.Kind == models.KindResource { - for _, r := range ro.Rules { - if len(r.Verbs) == 1 && r.Verbs[0] == models.MethodAll { - r.Verbs = AllActions + for _, policy := range policies { + verbAttrMap := make(map[string]sets.String) + resourceVerbs := make(map[string]sets.String) + for _, r := range policy.Rules { + for _, verb := range r.Verbs { + if verbs, ok := resourceVerbs[r.Resources[0]]; ok { + for _, v := range r.Verbs { + verbs.Insert(v) + } + resourceVerbs[r.Resources[0]] = verbs + } else { + verbSet := sets.String{} + for _, v := range r.Verbs { + verbSet.Insert(v) + } + resourceVerbs[r.Resources[0]] = verbSet } - for _, res := range r.Resources { - if mapping, ok := mappings[res]; !ok { - continue - } else { - for _, v := range r.Verbs { - opaRole.Rules = append(opaRole.Rules, mapping[v]...) - } + if attrs, ok := verbAttrMap[verb]; ok { + for _, attribute := range r.MatchAttributes { + attrs.Insert(attribute.Key + "&&" + attribute.Value) } - + verbAttrMap[verb] = attrs + } else { + attrSet := sets.String{} + for _, attribute := range r.MatchAttributes { + attrSet.Insert(attribute.Key + "&&" + attribute.Value) + } + verbAttrMap[verb] = attrSet } } - } else { - for _, r := range ro.Rules { + } + + //TODO - mouuii change role model to policy model + opaRole := &role{Name: policy.Name, Namespace: policy.Namespace} + for resource, verbs := range resourceVerbs { + ruleList := resourceMappings.GetPolicyRules(resource, verbs.List(), verbAttrMap) + opaRole.Rules = append(opaRole.Rules, ruleList...) + } + for _, r := range policy.Rules { + if r.Kind != models.KindResource { if len(r.Verbs) == 1 && r.Verbs[0] == models.MethodAll { r.Verbs = AllMethods } for _, v := range r.Verbs { for _, endpoint := range r.Resources { - opaRole.Rules = append(opaRole.Rules, &rule{Method: v, Endpoint: endpoint}) + opaRole.Rules = append(opaRole.Rules, &Rule{Method: v, Endpoint: endpoint}) } } } } - sort.Sort(opaRole.Rules) data.Roles = append(data.Roles, opaRole) } - sort.Sort(data.Roles) - return data } -func generateOPARoleBindings(bindings []*models.RoleBinding) *opaRoleBindings { +func generateOPABindings(rbs []*models.RoleBinding, pbs []*models.PolicyBinding) *opaRoleBindings { data := &opaRoleBindings{} - userRoleMap := make(map[string][]*roleRef) + userRoleMap := make(map[string]map[string][]*roleRef) - for _, rb := range bindings { + for _, rb := range rbs { for _, s := range rb.Subjects { if s.Kind == models.UserKind { - userRoleMap[s.Name] = append(userRoleMap[s.Name], &roleRef{Name: rb.RoleRef.Name, Namespace: rb.RoleRef.Namespace}) + if _, ok := userRoleMap[s.UID]; !ok { + userRoleMap[s.UID] = make(map[string][]*roleRef) + } + userRoleMap[s.UID][rb.Namespace] = append(userRoleMap[s.UID][rb.Namespace], &roleRef{Name: rb.RoleRef.Name, Namespace: rb.RoleRef.Namespace}) } } } - for k, v := range userRoleMap { - sort.Sort(roleRefs(v)) - data.RoleBindings = append(data.RoleBindings, &roleBinding{User: k, RoleRefs: v}) + for u, nb := range userRoleMap { + var bindingsData []*binding + for n, b := range nb { + sort.Sort(roleRefs(b)) + bindingsData = append(bindingsData, &binding{Namespace: n, RoleRefs: b}) + } + sort.Sort(bindings(bindingsData)) + data.RoleBindings = append(data.RoleBindings, &roleBinding{UID: u, Bindings: bindingsData}) } sort.Sort(data.RoleBindings) + userPolicyMap := make(map[string]map[string][]*roleRef) + + for _, rb := range pbs { + for _, s := range rb.Subjects { + if s.Kind == models.UserKind { + if _, ok := userPolicyMap[s.UID]; !ok { + userPolicyMap[s.UID] = make(map[string][]*roleRef) + } + userPolicyMap[s.UID][rb.Namespace] = append(userPolicyMap[s.UID][rb.Namespace], &roleRef{Name: rb.PolicyRef.Name, Namespace: rb.PolicyRef.Namespace}) + } + } + } + + for u, nb := range userPolicyMap { + var bindingsData []*bindingPolicy + for n, b := range nb { + sort.Sort(roleRefs(b)) + bindingsData = append(bindingsData, &bindingPolicy{Namespace: n, RoleRefs: b}) + } + sort.Sort(bindingPolicys(bindingsData)) + data.PolicyBindings = append(data.PolicyBindings, &policyBinding{UID: u, Bindings: bindingsData}) + } + + sort.Sort(data.PolicyBindings) + return data } -func generateOPAExemptionURLs() *exemptionURLs { - data := &exemptionURLs{} +type ExemptionURLs struct { + Public Rules `json:"public"` // public urls are not controlled by AuthN and AuthZ + Privileged Rules `json:"privileged"` // privileged urls can only be visited by system admins + Registered Rules `json:"registered"` // registered urls are the entire list of urls which are controlled by AuthZ, which means that if an url is not in this list, it is not controlled by AuthZ +} + +func generateOPAExemptionURLs(policies []*models.PolicyMeta) *ExemptionURLs { + data := &ExemptionURLs{} - for _, r := range globalURLs { + for _, r := range yamlconfig.GetExemptionsUrls().Public { if len(r.Methods) == 1 && r.Methods[0] == models.MethodAll { r.Methods = AllMethods } for _, method := range r.Methods { - for _, endpoint := range r.Endpoints { - data.Global = append(data.Global, &rule{Method: method, Endpoint: endpoint}) - } + data.Public = append(data.Public, &Rule{Method: method, Endpoint: r.Endpoint}) + } } - sort.Sort(data.Global) + sort.Sort(data.Public) - for _, r := range namespacedURLs { + for _, r := range yamlconfig.GetExemptionsUrls().SystemAdmin { if len(r.Methods) == 1 && r.Methods[0] == models.MethodAll { r.Methods = AllMethods } for _, method := range r.Methods { - for _, endpoint := range r.Endpoints { - data.Namespaced = append(data.Namespaced, &rule{Method: method, Endpoint: endpoint}) + data.Privileged = append(data.Privileged, &Rule{Method: method, Endpoint: r.Endpoint}) + } + } + sort.Sort(data.Privileged) + + resourceMappings := getResourceActionMappings(false, policies) + for _, resourceMappings := range resourceMappings { + for _, rs := range resourceMappings { + for _, r := range rs { + data.Registered = append(data.Registered, r) } } } - sort.Sort(data.Namespaced) + adminURLs := append(yamlconfig.GetExemptionsUrls().SystemAdmin, yamlconfig.GetExemptionsUrls().ProjectAdmin...) - for _, r := range publicURLs { + for _, r := range adminURLs { if len(r.Methods) == 1 && r.Methods[0] == models.MethodAll { r.Methods = AllMethods } for _, method := range r.Methods { - for _, endpoint := range r.Endpoints { - data.Public = append(data.Public, &rule{Method: method, Endpoint: endpoint}) - } + data.Registered = append(data.Registered, &Rule{Method: method, Endpoint: r.Endpoint}) } } - sort.Sort(data.Public) + sort.Sort(data.Registered) return data } -func generateOPAManifest() *opaManifest { - return &opaManifest{ - Revision: uuid.New().String(), - Roots: []string{""}, - } -} - -func generateOPAPolicy() []byte { +func generateOPAPolicyRego() []byte { return authz } func GenerateOPABundle() error { - log.Info("Generating OPA bundle") - defer log.Info("OPA bundle is generated") - - roles, err := mongodb.NewRoleColl().List() + rs, err := mongodb.NewRoleColl().List() if err != nil { log.Errorf("Failed to list roles, err: %s", err) } - bindings, err := mongodb.NewRoleBindingColl().List() + bs, err := mongodb.NewRoleBindingColl().List() if err != nil { log.Errorf("Failed to list roleBindings, err: %s", err) } - data := &opaData{ - {data: generateOPAManifest(), path: manifestPath}, - {data: generateOPAPolicy(), path: policyPath}, - {data: generateOPARoles(roles), path: rolesPath}, - {data: generateOPARoleBindings(bindings), path: rolebindingsPath}, - {data: generateOPAExemptionURLs(), path: exemptionPath}, + pbs, err := mongodb.NewPolicyBindingColl().List() + if err != nil { + log.Errorf("Failed to list roleBindings, err: %s", err) } - return data.save() -} - -func GetRevision() string { - data, err := afero.ReadFile(cacheFS, manifestPath) + pms, err := mongodb.NewPolicyMetaColl().List() + if err != nil { + log.Errorf("Failed to list policyMetas, err: %s", err) + } + policies, err := mongodb.NewPolicyColl().List() if err != nil { - log.Errorf("Failed to read manifest, err: %s", err) - return "" + log.Errorf("Failed to list policies, err: %s", err) + } + + bundle := &opa.Bundle{ + Data: []*opa.DataSpec{ + {Data: generateOPAPolicyRego(), Path: policyRegoPath}, + {Data: generateOPARoles(rs, pms), Path: rolesPath}, + {Data: generateOPAPolicies(policies, pms), Path: policiesPath}, + {Data: generateOPABindings(bs, pbs), Path: bindingsPath}, + {Data: generateOPAExemptionURLs(pms), Path: exemptionsPath}, + {Data: generateResourceBundle(), Path: resourcesPath}, + }, + Roots: []string{policyRoot, rolesRoot, rolebindingsRoot, exemptionsRoot, resourcesRoot, policiesRoot}, } - mf := &opaManifest{} - if err = json.Unmarshal(data, mf); err != nil { - log.Errorf("Failed to Unmarshal manifest, err: %s", err) - return "" + hash, err := bundle.Rehash() + if err != nil { + log.Errorf("Failed to calculate bundle hash, err: %s", err) + return err } + revision = hash + + return bundle.Save(config.DataPath()) +} - return mf.Revision +func GetRevision() string { + return revision } diff --git a/pkg/microservice/policy/core/service/bundle/opa_bundle_test.go b/pkg/microservice/policy/core/service/bundle/opa_bundle_test.go index 62a5e86f07d60c5aea65cc35751fa8a3f12159c2..80c131c6760de5cd81ae1364f25afea7eec21e02 100644 --- a/pkg/microservice/policy/core/service/bundle/opa_bundle_test.go +++ b/pkg/microservice/policy/core/service/bundle/opa_bundle_test.go @@ -59,11 +59,11 @@ var testBinding1 = ` "subjects": [ { "kind": "user", - "name": "alice" + "uid": "alice" }, { "kind": "user", - "name": "bob" + "uid": "bob" } ], "roleRef": { @@ -80,7 +80,7 @@ var testBinding2 = ` "subjects": [ { "kind": "user", - "name": "alice" + "uid": "alice" } ], "roleRef": { @@ -149,7 +149,7 @@ var expectOPARoleBindings = ` { "role_bindings": [ { - "user": "alice", + "uid": "alice", "role_refs": [ { "name": "author", @@ -162,7 +162,7 @@ var expectOPARoleBindings = ` ] }, { - "user": "bob", + "uid": "bob", "role_refs": [ { "name": "superuser", @@ -193,7 +193,7 @@ var _ = Describe("Testing generate OPA roles and bindings", func() { }) It("should work as expected", func() { - data := generateOPARoles(testRoles) + data := generateOPARoles(testRoles, nil) actual, err := json.MarshalIndent(data, "", " ") Expect(err).ShouldNot(HaveOccurred()) Expect(string(actual)).To(Equal(strings.TrimSpace(expectOPARoles))) diff --git a/pkg/microservice/policy/core/service/bundle/opa_policy.go b/pkg/microservice/policy/core/service/bundle/opa_policy.go index 33af2e180401a7197cf463171a20e0c515cf87b1..8b1cd4e3c94d6b207fc0a4c02a67f683de76af59 100644 --- a/pkg/microservice/policy/core/service/bundle/opa_policy.go +++ b/pkg/microservice/policy/core/service/bundle/opa_policy.go @@ -18,8 +18,6 @@ package bundle import ( _ "embed" - - _ "github.com/koderover/zadig/pkg/util/testing" ) //go:embed rego/authz.rego diff --git a/pkg/microservice/policy/core/service/bundle/rego/authz.rego b/pkg/microservice/policy/core/service/bundle/rego/authz.rego index b3119d416e8d984e8d9d58adeac3ec34b3d5f933..60ae61ed8e800f3f95145fc002bf051a0e0a3ad7 100644 --- a/pkg/microservice/policy/core/service/bundle/rego/authz.rego +++ b/pkg/microservice/policy/core/service/bundle/rego/authz.rego @@ -2,6 +2,61 @@ package rbac import input.attributes.request.http as http_request +# Policy rule definitions in rbac style, which is consumed by OPA server. +# you can use it to: +# 1. decide if a request is allowed and get status code and additional headers(if any) by querying: rbac.response +# 2. get all visible projects for an authenticated user by querying: rbac.user_visible_projects +# 3. get all allowed projects for a certain action(method+endpoint) for an authenticated user by querying: rbac.user_allowed_projects +# 4. check if a user is system admin by querying: rbac.user_is_admin +# 5. check if a user is project admin by querying: rbac.user_is_project_admin + +default response = { + "allowed": false, + "http_status": 403 +} + +response = r { + not is_authenticated + not url_is_public + r := { + "allowed": false, + "http_status": 401 + } +} + +response = r { + allow + roles := all_roles + r := { + "allowed": true, + "headers": { + "Roles": json.marshal(roles), + } + } +} + +# response for resource filtering, all allowed resources IDs will be returned in headers +response = r { + is_authenticated + not allow + rule_is_matched_for_filtering + roles := all_roles + role_resource := user_role_allowed_resources + policy_resource := user_policy_allowed_resources + resource := role_resource | policy_resource + policy_rule := user_matched_policy_rule + role_rule := user_matched_role_rule + rule := policy_rule | role_rule + r := { + "allowed": true, + "headers": { + "Resources": json.marshal(resource), + "Roles": json.marshal(roles), + "Rules": json.marshal(rule) + } + } +} + # By default, deny requests. default allow = false @@ -10,107 +65,410 @@ allow { url_is_public } -# Allow all valid users to visit exempted urls. allow { + is_authenticated + access_is_granted +} + +# Allow all valid users to visit exempted urls. +access_is_granted { url_is_exempted - claims.name != "" } # Allow admins to do anything. -allow { +access_is_granted { user_is_admin } # Allow project admins to do anything under the given project. -allow { +access_is_granted { + not url_is_privileged user_is_project_admin } # Allow the action if the user is granted permission to perform the action. -allow { - some grant - user_is_granted[grant] +access_is_granted { + not url_is_privileged + + some rule + + allowed_role_plain_rules[rule] + rule.method == http_request.method + glob.match(trim(rule.endpoint, "/"), ["/"], concat("/", input.parsed_path)) +} + +access_is_granted { + not url_is_privileged + + some rule + + allowed_policy_plain_rules[rule] + rule.method == http_request.method + glob.match(trim(rule.endpoint, "/"), ["/"], concat("/", input.parsed_path)) +} + +access_is_granted { + some rule + + allowed_role_attributive_rules[rule] + rule.method == http_request.method + glob.match(trim(rule.endpoint, "/"), ["/"], concat("/", input.parsed_path)) + + any_attribute_match(rule.matchAttributes, rule.resourceType, get_resource_id(rule.idRegex)) +} + +access_is_granted { + not url_is_privileged + + some rule + + allowed_system_role_plain_rules[rule] + rule.method == http_request.method + glob.match(trim(rule.endpoint, "/"), ["/"], concat("/", input.parsed_path)) +} + + +access_is_granted { + some rule + + allowed_policy_attributive_rules[rule] + rule.method == http_request.method + glob.match(trim(rule.endpoint, "/"), ["/"], concat("/", input.parsed_path)) + + any_attribute_match(rule.matchAttributes, rule.resourceType, get_resource_id(rule.idRegex)) +} + +rule_is_matched_for_filtering { + count(user_matched_role_rule_for_filtering) > 0 +} + +rule_is_matched_for_filtering { + count(user_matched_policy_rule_for_filtering) > 0 +} - grant.method == http_request.method - glob.match(trim(grant.endpoint, "/"), ["/"], concat("/", input.parsed_path)) +# get all resources which matches the attributes +user_role_allowed_resources[resourceID] { + some rule + + user_matched_role_rule_for_filtering[rule] + res := data.resources[rule.resourceType][_] + project_name_is_match(res) + attributes_match(rule.matchAttributes, res) + resourceID := res.resourceID +} + +user_policy_allowed_resources[resourceID] { + some rule + + user_matched_policy_rule_for_filtering[rule] + res := data.resources[rule.resourceType][_] + project_name_is_match(res) + attributes_match(rule.matchAttributes, res) + resourceID := res.resourceID +} + +attributes_match(attributes, res) { + count(attributes) == 0 +} + +attributes_match(attributes, res) { + attribute := attributes[_] + attribute_match(attribute, res) +} + +attribute_match(attribute, res) { + resValue := res.spec[_] + value_match(resValue, sprintf("%s:%s",[attribute.key,attribute.value])) +} + +value_match(resValue, value) { + resValue == value +} + +project_name_is_match(res) { + res.projectName != "" + res.projectName == project_name +} + +project_name_is_match(res) { + res.projectName == "" +} + +user_matched_policy_rule[rule] { + some rule + + allowed_policy_attributive_rules[rule] + + glob.match(trim(rule.endpoint, "/"), ["/"], concat("/", input.parsed_path)) + +} + +user_matched_role_rule[rule] { + some rule + + allowed_role_attributive_rules[rule] + + glob.match(trim(rule.endpoint, "/"), ["/"], concat("/", input.parsed_path)) + +} + +user_matched_policy_rule_for_filtering[rule] { + some rule + + allowed_policy_attributive_rules[rule] + rule.method == http_request.method + glob.match(trim(rule.endpoint, "/"), ["/"], concat("/", input.parsed_path)) + not rule.idRegex +} + +user_matched_role_rule_for_filtering[rule] { + some rule + + allowed_role_attributive_rules[rule] + rule.method == http_request.method + glob.match(trim(rule.endpoint, "/"), ["/"], concat("/", input.parsed_path)) + not rule.idRegex +} + + +any_attribute_match(attributes, resourceType, resourceID) { + res := data.resources[resourceType][_] + res.resourceID == resourceID + project_name_is_match(res) + + attributes_match(attributes, res) +} + + +get_resource_id(idRegex) = id { + output := regex.find_all_string_submatch_n(trim(idRegex, "/"), concat("/", input.parsed_path), -1) + count(output) == 1 + count(output[0]) == 2 + id := output[0][1] } user_is_admin { some role - allewed_roles[role] + all_roles[role] role.name == "admin" - role.namespace == "" + role.namespace == "*" } user_is_project_admin { some role - allewed_roles[role] + allowed_roles[role] - role.name == "admin" - role.namespace == projcet_name + role.name == "project-admin" + #role.namespace == project_name } +# public urls are visible for all users url_is_public { - data.exemptions.public[_].method == http_request.method - glob.match(trim(data.exemptions.public[_].endpoint, "/"), ["/"], concat("/", input.parsed_path)) + some i + data.exemptions.public[i].method == http_request.method + glob.match(trim(data.exemptions.public[i].endpoint, "/"), ["/"], concat("/", input.parsed_path)) } +# exempted urls are visible for all authenticated users url_is_exempted { - data.exemptions.global[_].method == http_request.method - glob.match(trim(data.exemptions.global[_].endpoint, "/"), ["/"], concat("/", input.parsed_path)) + not url_is_registered } -url_is_exempted { - data.exemptions.namespaced[_].method == http_request.method - glob.match(trim(data.exemptions.namespaced[_].endpoint, "/"), ["/"], concat("/", input.parsed_path)) - user_projects[_] == projcet_name +url_is_registered { + some i + data.exemptions.registered[i].method == http_request.method + glob.match(trim(data.exemptions.registered[i].endpoint, "/"), ["/"], concat("/", input.parsed_path)) +} + +# privileged urls are visible for system admins only +url_is_privileged { + some i + data.exemptions.privileged[i].method == http_request.method + glob.match(trim(data.exemptions.privileged[i].endpoint, "/"), ["/"], concat("/", input.parsed_path)) } -projcet_name := pn { +project_name := pn { pn := input.parsed_query.projectName[0] } -roles[role] { +# get all projects which are visible by current user +user_projects[project] { + some i + data.bindings.role_bindings[i].uid == claims.uid + project := data.bindings.role_bindings[i].bindings[_].namespace + project !="*" +} +user_projects[project] { + some i + data.bindings.policy_bindings[i].uid == claims.uid + project := data.bindings.policy_bindings[i].bindings[_].namespace + project !="*" +} + +# get all projects which are visible by all users (the user name is "*") +user_projects[project] { + some i + data.bindings.role_bindings[i].uid == "*" + project := data.bindings.role_bindings[i].bindings[_].namespace +} + +user_projects[project] { + some i + data.bindings.policy_bindings[i].uid == "*" + project := data.bindings.policy_bindings[i].bindings[_].namespace +} + +# all projects which are allowed by current user +user_allowed_projects[project] { + some project + user_projects[project] + not user_is_admin + response.allowed with project_name as project +} + +# if user is system admin, return all projects +user_allowed_projects[project] { + project := "*" + user_is_admin +} + +# all projects which are visible by current user +user_visible_projects[project] { + some project + user_projects[project] + not user_is_admin +} + +# if user is system admin, return all projects +user_visible_projects[project] { + project := "*" + user_is_admin +} + +all_roles[role_ref] { + some i + data.bindings.role_bindings[i].uid == claims.uid + role_ref := data.bindings.role_bindings[i].bindings[j].role_refs[_] +} + + +# only roles under the given project are allowed +allowed_roles[role_ref] { some i - data.bindings.role_bindings[i].user == claims.name - role_refs := data.bindings.role_bindings[i].role_refs - role := role_refs[_] + some j + data.bindings.role_bindings[i].uid == claims.uid + data.bindings.role_bindings[i].bindings[j].namespace == project_name + role_ref := data.bindings.role_bindings[i].bindings[j].role_refs[_] } -# only global roles and roles under the given project are allowed. -allewed_roles[role_ref] { - role_ref := roles[_] - role_ref.namespace == "" +allowed_system_roles[role_ref]{ + some i + some j + data.bindings.role_bindings[i].uid == claims.uid + data.bindings.role_bindings[i].bindings[j].namespace == "*" + role_ref := data.bindings.role_bindings[i].bindings[j].role_refs[_] } -allewed_roles[role_ref] { - role_ref := roles[_] - role_ref.namespace == projcet_name +# if the proejct is visible by all users (the user name is "*"), the bound roles are also allowed +allowed_roles[role_ref] { + some i + some j + data.bindings.role_bindings[i].uid == "*" + data.bindings.role_bindings[i].bindings[j].namespace == project_name + role_ref := data.bindings.role_bindings[i].bindings[j].role_refs[_] } -user_is_granted[grant] { +allowed_policies[policy_ref] { + some i + some j + data.bindings.policy_bindings[i].uid == claims.uid + data.bindings.policy_bindings[i].bindings[j].namespace == project_name + policy_ref := data.bindings.policy_bindings[i].bindings[j].policy_refs[_] +} + +allowed_policies[policy_ref] { + some i + some j + data.bindings.policy_bindings[i].uid == "*" + data.bindings.policy_bindings[i].bindings[j].namespace == project_name + policy_ref := data.bindings.policy_bindings[i].bindings[j].policy_refs[_] +} + +allowed_role_rules[rule] { some role_ref - allewed_roles[role_ref] + allowed_roles[role_ref] some i data.roles.roles[i].name == role_ref.name data.roles.roles[i].namespace == role_ref.namespace - grant := data.roles.roles[i].rules[_] + rule := data.roles.roles[i].rules[_] } -# get all projects which are visible by current user -user_projects[project] { - project := roles[_].namespace - project != "" +allowed_system_role_rules[rule] { + some role_ref + allowed_system_roles[role_ref] + + some i + data.roles.roles[i].name == role_ref.name + data.roles.roles[i].namespace == "*" + rule := data.roles.roles[i].rules[_] +} + +allowed_policy_rules[rule] { + some policy_ref + allowed_policies[policy_ref] + + some i + data.policies.policies[i].name == policy_ref.name + data.policies.policies[i].namespace == policy_ref.namespace + rule := data.policies.policies[i].rules[_] +} + +allowed_policy_plain_rules[rule] { + rule := allowed_policy_rules[_] + not rule.matchAttributes + not rule.matchExpressions +} + +allowed_role_plain_rules[rule] { + rule := allowed_role_rules[_] + not rule.matchAttributes + not rule.matchExpressions +} + +allowed_system_role_plain_rules[rule] { + rule := allowed_system_role_rules[_] + not rule.matchAttributes + not rule.matchExpressions +} + +allowed_policy_attributive_rules[rule] { + rule := allowed_policy_rules[_] + rule.matchAttributes +} + +allowed_role_attributive_rules[rule] { + rule := allowed_role_rules[_] + rule.matchAttributes +} + +allowed_policy_attributive_rules[rule] { + rule := allowed_policy_rules[_] + rule.matchExpressions +} + +allowed_role_attributive_rules[rule] { + rule := allowed_role_rules[_] + rule.matchExpressions } claims := payload { - # TODO: Verify the signature on the Bearer token. The certificate can be + # Verify the signature on the Bearer token. The certificate can be # hardcoded into the policy, and it could also be loaded via data or # an environment variable. Environment variables can be accessed using # the `opa.runtime()` built-in function. - # io.jwt.verify_rs256(bearer_token, certificate) + io.jwt.verify_hs256(bearer_token, secret) # This statement invokes the built-in function `io.jwt.decode` passing the # parsed bearer_token as a parameter. The `io.jwt.decode` function returns an @@ -121,9 +479,15 @@ claims := payload { # In Rego, you can pattern match values using the `=` and `:=` operators. This # example pattern matches on the result to obtain the JWT payload. [_, payload, _] := io.jwt.decode(bearer_token) + + # it is not working, don't know why + # [valid, _, payload] := io.jwt.decode_verify(bearer_token, { + # "secret": secret, + # "alg": "alg", + # }) } -bearer_token := t { +bearer_token_in_header := t { # Bearer tokens are contained inside of the HTTP Authorization header. This rule # parses the header and extracts the Bearer token value. If no Bearer token is # provided, the `bearer_token` value is undefined. @@ -131,3 +495,26 @@ bearer_token := t { startswith(v, "Bearer ") t := substring(v, count("Bearer "), -1) } + +bearer_token = t { + t := bearer_token_in_header +} + +bearer_token = t { + not bearer_token_in_header + t := input.parsed_query.token[0] +} + +is_authenticated { + claims + claims.uid != "" + claims.exp > time.now_ns()/1000000000 +} + +envs := env { + env := opa.runtime()["env"] +} + +secret := s { + s := envs["SECRET_KEY"] +} diff --git a/pkg/microservice/policy/core/service/bundle/rego/test_data/bindings/data.json b/pkg/microservice/policy/core/service/bundle/rego/test_data/bindings/data.json index 0512dd84ff2fe92b545750d03ba5f88c52b22c3c..52782d075227a26755fcc824b962f1e52a142d00 100644 --- a/pkg/microservice/policy/core/service/bundle/rego/test_data/bindings/data.json +++ b/pkg/microservice/policy/core/service/bundle/rego/test_data/bindings/data.json @@ -1,25 +1,58 @@ { "role_bindings": [ { - "user": "alice", - "role_refs": [ + "uid": "2e6c5200-358f-11ec-981f-de2269351a1e", + "bindings": [ { - "name": "reader1", - "namespace": "project1" + "namespace": "test", + "role_refs": [ + { + "name": "test", + "namespace": "test" + }, + { + "name": "reader2", + "namespace": "" + } + ] }, { - "name": "reader", - "namespace": "project2" - }, + "namespace": "project2", + "role_refs": [ + { + "name": "reader", + "namespace": "project2" + }, + { + "name": "reader3", + "namespace": "project2" + } + ] + } + ] + }, + { + "uid": "*", + "bindings": [ { - "name": "reader3", - "namespace": "project2" + "namespace": "project3", + "role_refs": [ + { + "name": "view", + "namespace": "" + } + ] }, { - "name": "reader2", - "namespace": "" + "namespace": "project1", + "role_refs": [ + { + "name": "view", + "namespace": "" + } + ] } ] } ] -} \ No newline at end of file +} diff --git a/pkg/microservice/policy/core/service/bundle/rego/test_data/exemptions/data.json b/pkg/microservice/policy/core/service/bundle/rego/test_data/exemptions/data.json index 866aa979773828988d7fb19f1a18b21a8d520c50..ee052a0237f0a4e2f73fc578bfabb5b628ebc033 100644 --- a/pkg/microservice/policy/core/service/bundle/rego/test_data/exemptions/data.json +++ b/pkg/microservice/policy/core/service/bundle/rego/test_data/exemptions/data.json @@ -1,20 +1,18 @@ { - "global": [ + "public": [ { - "method": "GET", - "endpoint": "/version" + "method": "POST", + "endpoint": "/api/aslan/webhook" } ], - "namespaced": [ + "registered": [ { "method": "GET", - "endpoint": "/version1" - } - ], - "public": [ + "endpoint": "/api/aslan/environment/environments/?*" + }, { - "method": "POST", - "endpoint": "/api/aslan/webhook" + "method": "GET", + "endpoint": "/api/aslan/environment/environments" } ] } diff --git a/pkg/microservice/policy/core/service/bundle/rego/test_data/resources/data.json b/pkg/microservice/policy/core/service/bundle/rego/test_data/resources/data.json new file mode 100644 index 0000000000000000000000000000000000000000..0b331663bc82e17a1db6e75546ae86df85afba4c --- /dev/null +++ b/pkg/microservice/policy/core/service/bundle/rego/test_data/resources/data.json @@ -0,0 +1,22 @@ +{ + "Environment": [ + { + "projectName": "test", + "resourceID": "test", + "production": "true", + "production1": "true1" + }, + { + "projectName": "test", + "resourceID": "test1", + "production": "true", + "production1": "true1" + }, + { + "projectName": "test", + "resourceID": "test2", + "production": "false", + "production1": "true1" + } + ] +} diff --git a/pkg/microservice/policy/core/service/bundle/rego/test_data/roles/data.json b/pkg/microservice/policy/core/service/bundle/rego/test_data/roles/data.json index 68ce3d2fb15fce9d2ea39f2a57fe2344d20b0584..05f449ff859495e24799c59f55ae5662ff864b24 100644 --- a/pkg/microservice/policy/core/service/bundle/rego/test_data/roles/data.json +++ b/pkg/microservice/policy/core/service/bundle/rego/test_data/roles/data.json @@ -1,94 +1,35 @@ { "roles": [ { - "name": "author", - "namespace": "project2", + "name": "test", + "namespace": "test", "rules": [ { "method": "GET", - "endpoint": "/api/authors" + "endpoint": "/api/aslan/environment/environments/?*", + "resourceType": "Environment", + "idRegex": "/api/aslan/environment/environments/([\\w\\W]+?)$", + "matchAttributes": [ + { + "key": "production", + "value": "true" + }, + { + "key": "production1", + "value": "true1" + } + ] }, - { - "method": "POST", - "endpoint": "/api/authors" - }, - { - "method": "PUT", - "endpoint": "/api/authors" - }, - { - "method": "PATCH", - "endpoint": "/api/authors" - }, - { - "method": "DELETE", - "endpoint": "/api/authors" - } - ] - }, - { - "name": "reader", - "namespace": "project2", - "rules": [ - { - "method": "GET", - "endpoint": "/api/authors" - }, - { - "method": "GET", - "endpoint": "/api/articles" - }, - { - "method": "POST", - "endpoint": "/api/authors" - }, - { - "method": "POST", - "endpoint": "/api/articles" - } - ] - }, - { - "name": "reader1", - "namespace": "project1", - "rules": [ - { - "method": "GET", - "endpoint": "/api/authors" - }, - { - "method": "GET", - "endpoint": "/api/articles" - }, - { - "method": "POST", - "endpoint": "/api/authors" - }, - { - "method": "POST", - "endpoint": "/api/articles" - } - ] - }, - { - "name": "reader2", - "namespace": "project1", - "rules": [ { "method": "GET", - "endpoint": "/api/authors" - }, - { - "method": "GET", - "endpoint": "/api/articles" - }, - { - "method": "POST", - "endpoint": "/api/authors" - }, - { - "method": "POST", - "endpoint": "/api/articles" + "endpoint": "/api/aslan/environment/environments", + "resourceType": "Environment", + "matchAttributes": [ + { + "key": "production", + "value": "true" + } + ] } ] } diff --git a/pkg/microservice/policy/core/service/bundle/rego/test_input.json b/pkg/microservice/policy/core/service/bundle/rego/test_input.json index d6aad2a2669ffd0d8264c2000c9f31ef8bdb124b..5cd2935132c467747bfcc2797d2b9cff48ff27a0 100644 --- a/pkg/microservice/policy/core/service/bundle/rego/test_input.json +++ b/pkg/microservice/policy/core/service/bundle/rego/test_input.json @@ -19,7 +19,7 @@ "accept": "*/*", "accept-encoding": "gzip, deflate", "accept-language": "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2", - "authorization": "Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6ImFmMGNlZTQxOGY0NjJkNWMzMjYyNGRiY2FlN2NjM2E2MjFmMDlkMjMifQ.eyJpc3MiOiJodHRwOi8vZGV4LnRlc3QuOHNsYW4uY29tL2RleCIsInN1YiI6IkNpVXhNak0wTlRZM09EQXRaR0k0T0MwMFlqY3pMVGt3WVRrdE0yTmtNVFkyTVdZMU5EWTJFZ1ZzYjJOaGJBIiwiYXVkIjoiZXhhbXBsZS1hcHAiLCJleHAiOjE2MzM3ODQ2NTgsImlhdCI6MTYzMzY5ODI1OCwiYXRfaGFzaCI6IkhzV0trWFhucDAzZFh1aGxOc00ycUEiLCJjX2hhc2giOiJISjllRUhhMnNBTHY5elBVaERERDd3IiwiZW1haWwiOiJsb3V0ZXN0QGV4YW1wbGUuY29tIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJhbGljZSJ9.cAphZs4fv_ZxTNZpIziPi4btm50eYpSvXJGrmNP7TQilGKqkaIF6oroCHsdxuBm3pwIOI8dE4nws7Eogxdtx1AH6EQ3vWv8OoIW1TpCDQit4cX5MsLnEyzuS3r30QQXxi1x2veabsfQungudPivFr4_1xQcjho9G1Y3AGxEDU9zA0-r3U7g9M3lrcF4domaF1xzR1FT2TP0V2QVPCxOocPUl65P-Oxu7oRxpnv44d5ORvAuTuU5IrulZMs5NyaY145qzpBZSTU_x3xQAl4BE35XYh_w1wBqqSEpNbAum9VlpX9kaNrHc8B-jHw2cBRuEIyJU6CPC4g1wcYlxoCrCDg", + "authorization": "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoibG91dGVzdCIsImVtYWlsIjoibG91dGVzdEBleGFtcGxlLmNvbSIsInVpZCI6IjJlNmM1MjAwLTM1OGYtMTFlYy05ODFmLWRlMjI2OTM1MWExZSIsInByZWZlcnJlZF91c2VybmFtZSI6ImxvdXRlc3QiLCJmZWRlcmF0ZWRfY2xhaW1zIjp7ImNvbm5lY3Rvcl9pZCI6InN5c3RlbSIsInVzZXJfaWQiOiJsb3V0ZXN0In0sImF1ZCI6InphZGlnIiwiZXhwIjoxNzI0MDQ3NTMyfQ.H05PvCXmvPX4gpgMjfHDNUlS1s_EpYrqcFG61SYUJB8", "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:92.0) Gecko/20100101 Firefox/92.0", "x-forwarded-proto": "http", "x-request-id": "14404686-7a28-48a6-9be6-bdd6e119f81a" @@ -45,11 +45,13 @@ "parsed_body": null, "parsed_path": [ "api", - "articles" + "aslan", + "environment", + "environments" ], "parsed_query": { "projectName": [ - "project1" + "test" ] }, "truncated_body": false, diff --git a/pkg/microservice/policy/core/service/bundle/resource_bundle.go b/pkg/microservice/policy/core/service/bundle/resource_bundle.go new file mode 100644 index 0000000000000000000000000000000000000000..9fb5de8bc63bf57d59c62fb3d5f134bb8a672858 --- /dev/null +++ b/pkg/microservice/policy/core/service/bundle/resource_bundle.go @@ -0,0 +1,137 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bundle + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "sort" + + "sigs.k8s.io/yaml" + + "github.com/koderover/zadig/pkg/tool/httpclient" + "github.com/koderover/zadig/pkg/tool/log" +) + +const configPath = "/config/config.yaml" + +type ResourceBundleService struct { + Endpoint string `json:"endpoint"` + ResourceType string `json:"resourceType"` +} + +type Config struct { + Bundles []*ResourceBundleService `json:"bundles"` + Namespace string `json:"namespace"` +} + +type ResourceSpec struct { + ResourceID string `json:"resourceID"` + ProjectName string `json:"projectName"` + Spec []string `json:"spec"` +} + +type ResourceBundle map[string]resources + +type resources []*ResourceSpec + +func (o resources) Len() int { return len(o) } +func (o resources) Swap(i, j int) { o[i], o[j] = o[j], o[i] } +func (o resources) Less(i, j int) bool { + if o[i].ProjectName == o[j].ProjectName { + return o[i].ResourceID < o[j].ResourceID + } + return o[i].ProjectName < o[j].ProjectName +} + +func (r ResourceBundle) MarshalJSON() ([]byte, error) { + var keys []string + for k := range r { + keys = append(keys, k) + } + sort.Strings(keys) + + buf := &bytes.Buffer{} + buf.Write([]byte{'{'}) + for i, key := range keys { + value := r[key] + b, err := json.Marshal(&value) + if err != nil { + return nil, err + } + buf.WriteString(fmt.Sprintf("%q:", key)) + buf.Write(b) + if i < len(keys)-1 { + buf.Write([]byte{','}) + } + } + buf.Write([]byte{'}'}) + + return buf.Bytes(), nil +} + +func loadConfig() (*Config, error) { + data, err := os.ReadFile(configPath) + if err != nil { + log.Errorf("Failed to load config, err: %s", err) + return nil, err + } + + conf := &Config{} + if err = yaml.Unmarshal(data, conf); err != nil { + log.Errorf("Failed to unmarshal yaml, err: %s", err) + return nil, err + } + + return conf, nil +} + +func AppendOPAResources(res ResourceBundle, resourceType string, objs []*ResourceSpec) ResourceBundle { + if res == nil { + res = make(map[string]resources) + } + + sort.Sort(resources(objs)) + res[resourceType] = objs + + return res +} + +func generateResourceBundle() ResourceBundle { + var res ResourceBundle + + cfg, err := loadConfig() + if err != nil { + return nil + } + + cl := httpclient.New() + for _, bundleService := range cfg.Bundles { + objs := make([]*ResourceSpec, 0) + _, err := cl.Get(bundleService.Endpoint, httpclient.SetResult(&objs)) + if err != nil { + log.Warnf("Failed to get %s bundle, err: %s", bundleService.ResourceType, err) + continue + } + + res = AppendOPAResources(res, bundleService.ResourceType, objs) + } + + return res +} diff --git a/pkg/microservice/policy/core/service/bundle/resource_endpoint_mapping.go b/pkg/microservice/policy/core/service/bundle/resource_endpoint_mapping.go index 2b328dd4623a676e163c07326f977b526ed42225..5518ca78605749bc0224eb7192fad91e78ccc411 100644 --- a/pkg/microservice/policy/core/service/bundle/resource_endpoint_mapping.go +++ b/pkg/microservice/policy/core/service/bundle/resource_endpoint_mapping.go @@ -1,16 +1,141 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package bundle -var mappings = map[string]map[string][]*rule{ - "Workflow": workflowMapping, +import ( + "strings" + + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/koderover/zadig/pkg/microservice/policy/core/repository/models" +) + +type resourceActionMappings map[string]map[string][]*Rule + +func (m resourceActionMappings) GetRules(resource string, actions []string) []*Rule { + mappings, ok := m[resource] + if !ok { + return nil + } + + all := false + if len(actions) == 1 && actions[0] == models.MethodAll { + all = true + } + actionSet := sets.NewString(actions...) + var res []*Rule + for action, r := range mappings { + if all || actionSet.Has(action) { + for _, rr := range r { + rrr := &Rule{ + Method: rr.Method, + Endpoint: rr.Endpoint, + ResourceType: rr.ResourceType, + IDRegex: rr.IDRegex, + MatchAttributes: rr.MatchAttributes, + MatchExpressions: rr.MatchExpressions, + } + res = append(res, rrr) + } + } + } + + return res +} + +func (m resourceActionMappings) GetPolicyRules(resource string, actions []string, verbAttrMap map[string]sets.String) []*Rule { + mappings, ok := m[resource] + if !ok { + return nil + } + + all := false + if len(actions) == 1 && actions[0] == models.MethodAll { + all = true + } + actionSet := sets.NewString(actions...) + var res []*Rule + for action, r := range mappings { + var attr Attributes + if attrs, ok := verbAttrMap[action]; ok { + for _, s := range attrs.List() { + attrString := strings.Split(s, "&&") + attr = append(attr, &Attribute{ + Key: attrString[0], + Value: attrString[1], + }) + } + } + if all || actionSet.Has(action) { + for _, rr := range r { + attrSets := sets.String{} + rrAttr := attr + for _, attribute := range rrAttr { + attrSets.Insert(attribute.Key + "&&" + attribute.Value) + } + for _, attribute := range rr.MatchAttributes { + if !attrSets.Has(attribute.Key + "&&" + attribute.Value) { + rrAttr = append(rrAttr, attribute) + attrSets.Insert(attribute.Key + "&&" + attribute.Value) + } + } + rrr := &Rule{ + Method: rr.Method, + Endpoint: rr.Endpoint, + ResourceType: rr.ResourceType, + IDRegex: rr.IDRegex, + MatchAttributes: rrAttr, + MatchExpressions: rr.MatchExpressions, + } + res = append(res, rrr) + } + } + } + + return res } -var workflowMapping = map[string][]*rule{ - ActionCreate: { - {Method: MethodPost, Endpoint: "/api/aslan/workflow/workflow"}, - {Method: MethodPost, Endpoint: "/api/aslan/workflow/v2/pipelines"}, - }, - ActionDelete: { - {Method: MethodPost, Endpoint: "/api/aslan/workflow/workflow/?*"}, - {Method: MethodPost, Endpoint: "/api/aslan/workflow/v2/pipelines/?*"}, - }, +func getResourceActionMappings(isPolicy bool, policies []*models.PolicyMeta) resourceActionMappings { + data := make(resourceActionMappings) + for _, p := range policies { + if _, ok := data[p.Resource]; !ok { + data[p.Resource] = make(map[string][]*Rule) + } + + for _, r := range p.Rules { + for _, ar := range r.Rules { + var as []*Attribute + for _, a := range ar.MatchAttributes { + if a.Key == "production" && isPolicy { + continue + } + as = append(as, &Attribute{Key: a.Key, Value: a.Value}) + } + + data[p.Resource][r.Action] = append(data[p.Resource][r.Action], &Rule{ + Method: ar.Method, + Endpoint: ar.Endpoint, + ResourceType: ar.ResourceType, + IDRegex: ar.IDRegex, + MatchAttributes: as, + }) + } + } + } + + return data } diff --git a/pkg/microservice/policy/core/service/common.go b/pkg/microservice/policy/core/service/common.go new file mode 100644 index 0000000000000000000000000000000000000000..fbc1755219b29cbf757dedc22440c99387e169dc --- /dev/null +++ b/pkg/microservice/policy/core/service/common.go @@ -0,0 +1,32 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "github.com/koderover/zadig/pkg/microservice/policy/core/repository/models" +) + +type Rule struct { + Verbs []string `json:"verbs"` + Resources []string `json:"resources"` + Kind string `json:"kind"` + MatchAttributes []models.MatchAttribute `json:"match_attributes"` + RelatedResources []string `json:"related_resources"` +} + +const SystemScope = "*" +const PresetScope = "" diff --git a/pkg/microservice/policy/core/service/permission.go b/pkg/microservice/policy/core/service/permission.go new file mode 100644 index 0000000000000000000000000000000000000000..48efdc193f466f845935666d374ed2adfbebb32e --- /dev/null +++ b/pkg/microservice/policy/core/service/permission.go @@ -0,0 +1,367 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "fmt" + "strings" + + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/label/config" + "github.com/koderover/zadig/pkg/microservice/policy/core/repository/models" + "github.com/koderover/zadig/pkg/microservice/policy/core/repository/mongodb" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/client/label" +) + +func GetUserRulesByProject(uid string, projectName string, log *zap.SugaredLogger) (*GetUserRulesByProjectResp, error) { + roleBindings, err := mongodb.NewRoleBindingColl().ListBy(projectName, uid) + if err != nil { + return nil, err + } + allUserRoleBingdins, err := mongodb.NewRoleBindingColl().ListBy(projectName, "*") + if err != nil { + return nil, err + } + roleBindings = append(roleBindings, allUserRoleBingdins...) + roles, err := ListUserAllRolesByRoleBindings(roleBindings) + if err != nil { + return nil, err + } + roleMap := make(map[string]*models.Role) + for _, role := range roles { + roleMap[role.Name] = role + } + isSystemAdmin := false + isProjectAdmin := false + projectVerbSet := sets.NewString() + for _, rolebinding := range roleBindings { + if rolebinding.RoleRef.Name == string(setting.SystemAdmin) { + isSystemAdmin = true + continue + } else if rolebinding.RoleRef.Name == string(setting.ProjectAdmin) { + isProjectAdmin = true + continue + } + var role *models.Role + if roleRef, ok := roleMap[rolebinding.RoleRef.Name]; ok { + role = roleRef + } else { + log.Errorf("roleMap has no role:%s", rolebinding.RoleRef.Name) + return nil, fmt.Errorf("roleMap has no role:%s", rolebinding.RoleRef.Name) + } + for _, rule := range role.Rules { + var ruleVerbs []string + if rule.Resources[0] == "ProductionEnvironment" { + for _, verb := range rule.Verbs { + ruleVerbs = append(ruleVerbs, "production:"+verb) + } + } else { + ruleVerbs = rule.Verbs + } + projectVerbSet.Insert(ruleVerbs...) + } + } + + policyBindings, err := mongodb.NewPolicyBindingColl().ListBy(projectName, uid) + if err != nil { + return nil, err + } + policies, err := ListUserAllPoliciesByPolicyBindings(policyBindings) + if err != nil { + return nil, err + } + policyMap := make(map[string]*models.Policy) + for _, policy := range policies { + policyMap[policy.Name] = policy + } + labelVerbMap := make(map[string][]string) + + for _, policyBinding := range policyBindings { + var policy *models.Policy + if policyRef, ok := policyMap[policyBinding.PolicyRef.Name]; ok { + policy = policyRef + } else { + log.Errorf("policyMap has no policy:%s", policyBinding.PolicyRef.Name) + return nil, fmt.Errorf("policyMap has no policy:%s", policyBinding.PolicyRef.Name) + } + + for _, rule := range policy.Rules { + for _, matchAttribute := range rule.MatchAttributes { + labelKeyKey := rule.Resources[0] + ":" + matchAttribute.Key + ":" + matchAttribute.Value + if verbs, ok := labelVerbMap[labelKeyKey]; ok { + verbsSet := sets.NewString(verbs...) + verbsSet.Insert(rule.Verbs...) + labelVerbMap[labelKeyKey] = verbsSet.List() + } else { + labelVerbMap[labelKeyKey] = rule.Verbs + } + } + } + } + var labels []label.Label + for labelKey, _ := range labelVerbMap { + keySplit := strings.Split(labelKey, ":") + labels = append(labels, label.Label{ + Type: keySplit[0], + Key: keySplit[1], + Value: keySplit[2], + }) + } + req := label.ListResourcesByLabelsReq{ + LabelFilters: labels, + } + resp, err := label.New().ListResourcesByLabels(req) + if err != nil { + return nil, err + } + environmentVerbMap := make(map[string][]string) + workflowVerbMap := make(map[string][]string) + for labelKey, resources := range resp.Resources { + for _, resource := range resources { + if verbs, ok := labelVerbMap[resource.Type+":"+labelKey]; ok { + if resource.Type == string(config.ResourceTypeEnvironment) { + if resourceVerbs, rOK := environmentVerbMap[resource.Name]; rOK { + verbSet := sets.NewString(resourceVerbs...) + verbSet.Insert(verbs...) + environmentVerbMap[resource.Name] = verbSet.List() + } else { + environmentVerbMap[resource.Name] = verbs + } + } + if resource.Type == string(config.ResourceTypeWorkflow) { + if resourceVerbs, rOK := workflowVerbMap[resource.Name]; rOK { + verbSet := sets.NewString(resourceVerbs...) + verbSet.Insert(verbs...) + workflowVerbMap[resource.Name] = verbSet.List() + } else { + workflowVerbMap[resource.Name] = verbs + } + } + } else { + log.Warnf("labelVerbMap key:%s not exist", resource.Type+":"+labelKey) + } + } + } + return &GetUserRulesByProjectResp{ + IsSystemAdmin: isSystemAdmin, + ProjectVerbs: projectVerbSet.List(), + IsProjectAdmin: isProjectAdmin, + EnvironmentVerbsMap: environmentVerbMap, + WorkflowVerbsMap: workflowVerbMap, + }, nil +} + +type GetUserRulesByProjectResp struct { + IsSystemAdmin bool `json:"is_system_admin"` + IsProjectAdmin bool `json:"is_project_admin"` + ProjectVerbs []string `json:"project_verbs"` + WorkflowVerbsMap map[string][]string `json:"workflow_verbs_map"` + EnvironmentVerbsMap map[string][]string `json:"environment_verbs_map"` +} + +type GetUserRulesResp struct { + IsSystemAdmin bool `json:"is_system_admin"` + ProjectAdminList []string `json:"project_admin_list"` + ProjectVerbMap map[string][]string `json:"project_verb_map"` + SystemVerbs []string `json:"system_verbs"` +} + +func GetUserRules(uid string, log *zap.SugaredLogger) (*GetUserRulesResp, error) { + roleBindings, err := mongodb.NewRoleBindingColl().ListRoleBindingsByUIDs([]string{uid, "*"}) + if err != nil { + log.Errorf("ListRoleBindingsByUIDs err:%s") + return &GetUserRulesResp{}, err + } + if len(roleBindings) == 0 { + log.Info("rolebindings == 0") + return &GetUserRulesResp{}, nil + } + roles, err := ListUserAllRolesByRoleBindings(roleBindings) + if err != nil { + log.Errorf("ListUserAllRolesByRoleBindings err:%s", err) + return &GetUserRulesResp{}, err + } + roleMap := make(map[string]*models.Role) + for _, role := range roles { + roleMap[role.Name] = role + } + isSystemAdmin := false + projectAdminSet := sets.NewString() + projectVerbMap := make(map[string][]string) + systemVerbSet := sets.NewString() + for _, rolebinding := range roleBindings { + if rolebinding.RoleRef.Name == string(setting.SystemAdmin) { + isSystemAdmin = true + continue + } else if rolebinding.RoleRef.Name == string(setting.ProjectAdmin) { + projectAdminSet.Insert(rolebinding.Namespace) + continue + } + var role *models.Role + if roleRef, ok := roleMap[rolebinding.RoleRef.Name]; ok { + role = roleRef + } else { + log.Errorf("roleMap has no role:%s", rolebinding.RoleRef.Name) + return nil, fmt.Errorf("roleMap has no role:%s", rolebinding.RoleRef.Name) + } + if rolebinding.Namespace == "*" { + for _, rule := range role.Rules { + systemVerbSet.Insert(rule.Verbs...) + } + } else { + if verbs, ok := projectVerbMap[rolebinding.Namespace]; ok { + verbSet := sets.NewString(verbs...) + for _, rule := range role.Rules { + var ruleVerbs []string + if rule.Resources[0] == "ProductionEnvironment" { + for _, verb := range rule.Verbs { + ruleVerbs = append(ruleVerbs, "production:"+verb) + } + } else { + ruleVerbs = rule.Verbs + } + verbSet.Insert(ruleVerbs...) + } + projectVerbMap[rolebinding.Namespace] = verbSet.List() + + } else { + verbSet := sets.NewString() + for _, rule := range role.Rules { + var ruleVerbs []string + if rule.Resources[0] == "ProductionEnvironment" { + for _, verb := range rule.Verbs { + ruleVerbs = append(ruleVerbs, "production:"+verb) + } + } else { + ruleVerbs = rule.Verbs + } + verbSet.Insert(ruleVerbs...) + } + projectVerbMap[rolebinding.Namespace] = verbSet.List() + } + } + } + return &GetUserRulesResp{ + IsSystemAdmin: isSystemAdmin, + ProjectVerbMap: projectVerbMap, + SystemVerbs: systemVerbSet.List(), + ProjectAdminList: projectAdminSet.List(), + }, nil +} + +//GetResourcesPermission get resources action list for frontend to show icon +func GetResourcesPermission(uid string, projectName string, resourceType string, resources []string, logger *zap.SugaredLogger) (map[string][]string, error) { + // 1. get all policyBindings + policyBindings, err := ListPolicyBindings(projectName, uid, logger) + if err != nil { + logger.Errorf("ListPolicyBindings err:%s", err) + return nil, err + } + var policies []*Policy + for _, v := range policyBindings { + policy, err := GetPolicy(projectName, v.Policy, logger) + if err != nil { + logger.Warnf("GetPolicy err:%s", err) + continue + } + policies = append(policies, policy) + } + + queryResourceSet := sets.NewString(resources...) + resourceM := make(map[string]sets.String) + for _, v := range resources { + resourceM[v] = sets.NewString() + } + for _, policy := range policies { + for _, rule := range policy.Rules { + if rule.Resources[0] == resourceType { + for _, resource := range rule.RelatedResources { + if queryResourceSet.Has(resource) { + resourceM[resource] = resourceM[resource].Insert(rule.Verbs...) + } + } + } + + } + } + // 2. get all roleBindings + roleBindings, err := ListUserAllRoleBindings(projectName, uid) + if err != nil { + logger.Errorf("ListUserAllRoleBindings err:%s", err) + return nil, err + } + roles, err := ListUserAllRolesByRoleBindings(roleBindings) + if err != nil { + logger.Errorf("ListUserAllRolesByRoleBindings err:%s", err) + return nil, err + } + for _, role := range roles { + if role.Name == string(setting.SystemAdmin) || role.Name == string(setting.ProjectAdmin) { + for k, _ := range resourceM { + resourceM[k] = sets.NewString("*") + } + break + } + for _, rule := range role.Rules { + if rule.Resources[0] == resourceType && resourceType != string(config.ResourceTypeEnvironment) { + for k, v := range resourceM { + resourceM[k] = v.Insert(rule.Verbs...) + } + } + if (rule.Resources[0] == "Environment" || rule.Resources[0] == "ProductionEnvironment") && resourceType == string(config.ResourceTypeEnvironment) { + var rs []label.Resource + for _, v := range resources { + r := label.Resource{ + Name: v, + ProjectName: projectName, + Type: string(config.ResourceTypeEnvironment), + } + rs = append(rs, r) + } + + labelRes, err := label.New().ListLabelsByResources(label.ListLabelsByResourcesReq{rs}) + if err != nil { + continue + } + for _, resource := range resources { + resourceKey := fmt.Sprintf("%s-%s-%s", config.ResourceTypeEnvironment, projectName, resource) + if labels, ok := labelRes.Labels[resourceKey]; ok { + for _, label := range labels { + if label.Key == "production" { + if rule.Resources[0] == "Environment" && label.Value == "false" { + resourceM[resource] = resourceM[resource].Insert(rule.Verbs...) + } + if rule.Resources[0] == "ProductionEnvironment" && label.Value == "true" { + resourceM[resource] = resourceM[resource].Insert(rule.Verbs...) + } + } + } + } + } + + } + } + } + resourceRes := make(map[string][]string) + for k, v := range resourceM { + resourceRes[k] = v.List() + } + return resourceRes, nil +} diff --git a/pkg/microservice/policy/core/service/policy.go b/pkg/microservice/policy/core/service/policy.go new file mode 100644 index 0000000000000000000000000000000000000000..e1b5d22d4f2749163ce5be430ed793f01aa84fab --- /dev/null +++ b/pkg/microservice/policy/core/service/policy.go @@ -0,0 +1,255 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "fmt" + "strings" + + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/label/service" + "github.com/koderover/zadig/pkg/microservice/policy/core/repository/models" + "github.com/koderover/zadig/pkg/microservice/policy/core/repository/mongodb" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/client/label" +) + +type Policy struct { + Name string `json:"name"` + Description string `json:"description"` + UpdateTime int64 `json:"update_time"` + Rules []*Rule `json:"rules,omitempty"` +} + +type ListPolicyResp struct { + Policies []Policy `json:"policies"` +} + +func CreatePolicy(ns string, policy *Policy, _ *zap.SugaredLogger) error { + obj := &models.Policy{ + Name: policy.Name, + Namespace: ns, + Type: setting.ResourceTypeSystem, + } + + for _, r := range policy.Rules { + obj.Rules = append(obj.Rules, &models.Rule{ + Verbs: r.Verbs, + Kind: r.Kind, + Resources: r.Resources, + MatchAttributes: r.MatchAttributes, + }) + } + return mongodb.NewPolicyColl().Create(obj) +} + +func CreatePolicies(ns string, policies []*Policy, _ *zap.SugaredLogger) error { + var objs []*models.Policy + for _, policy := range policies { + obj := &models.Policy{ + Name: policy.Name, + Namespace: ns, + Description: policy.Description, + UpdateTime: policy.UpdateTime, + Type: setting.ResourceTypeSystem, + } + + for _, r := range policy.Rules { + obj.Rules = append(obj.Rules, &models.Rule{ + Verbs: r.Verbs, + Kind: r.Kind, + Resources: r.Resources, + MatchAttributes: r.MatchAttributes, + }) + } + objs = append(objs, obj) + } + return mongodb.NewPolicyColl().BulkCreate(objs) +} + +func UpdatePolicy(ns string, policy *Policy, log *zap.SugaredLogger) error { + obj := &models.Policy{ + Name: policy.Name, + Namespace: ns, + UpdateTime: policy.UpdateTime, + Description: policy.Description, + } + + for _, r := range policy.Rules { + obj.Rules = append(obj.Rules, &models.Rule{ + Verbs: r.Verbs, + Kind: r.Kind, + Resources: r.Resources, + MatchAttributes: r.MatchAttributes, + }) + } + return mongodb.NewPolicyColl().UpdatePolicy(obj) +} + +func UpdateOrCreatePolicy(ns string, policy *Policy, _ *zap.SugaredLogger) error { + obj := &models.Policy{ + Name: policy.Name, + Namespace: ns, + UpdateTime: policy.UpdateTime, + Description: policy.Description, + } + + for _, r := range policy.Rules { + obj.Rules = append(obj.Rules, &models.Rule{ + Verbs: r.Verbs, + Kind: r.Kind, + Resources: r.Resources, + MatchAttributes: r.MatchAttributes, + }) + } + return mongodb.NewPolicyColl().UpdateOrCreate(obj) +} + +func ListPolicies(projectName string, _ *zap.SugaredLogger) ([]*Policy, error) { + var policies []*Policy + projectPolicies, err := mongodb.NewPolicyColl().ListBy(projectName) + if err != nil { + return nil, err + } + for _, v := range projectPolicies { + policies = append(policies, &Policy{ + Name: v.Name, + Description: v.Description, + UpdateTime: v.UpdateTime, + }) + } + return policies, nil +} + +func GetPolicies(names string, log *zap.SugaredLogger) ([]*Policy, error) { + splitedNames := strings.Split(names, ",") + policies, err := mongodb.NewPolicyColl().GetByNames(splitedNames) + if err != nil { + return nil, err + } + var res []*Policy + for _, policy := range policies { + policy, err := buildPolicy(policy) + if err != nil { + log.Errorf("buildPolicy error:%s", err) + return nil, err + } + res = append(res, policy) + } + + return res, nil +} + +func buildPolicy(r *models.Policy) (*Policy, error) { + res := &Policy{ + Name: r.Name, + } + var labels []label.Label + labelSet := sets.NewString() + for _, ru := range r.Rules { + res.Rules = append(res.Rules, &Rule{ + Verbs: ru.Verbs, + Kind: ru.Kind, + Resources: ru.Resources, + MatchAttributes: ru.MatchAttributes, + }) + for _, ma := range ru.MatchAttributes { + labelString := service.BuildLabelString(ma.Key, ma.Value) + if !labelSet.Has(labelString) { + labelSet.Insert(labelString) + labels = append(labels, label.Label{ + Key: ma.Key, + Value: ma.Value, + }) + } + } + } + req := label.ListResourcesByLabelsReq{ + LabelFilters: labels, + } + labelClient := label.New() + resp, err := labelClient.ListResourcesByLabels(req) + if err != nil { + return nil, err + } + for i, rule := range res.Rules { + var relatedResources []string + for _, ma := range rule.MatchAttributes { + labelString := service.BuildLabelString(ma.Key, ma.Value) + if resources, ok := resp.Resources[labelString]; ok { + for _, resource := range resources { + if resource.Type == rule.Resources[0] { + relatedResources = append(relatedResources, resource.Name) + } + } + } + } + res.Rules[i].RelatedResources = relatedResources + } + return res, nil +} + +func GetPolicy(ns, name string, log *zap.SugaredLogger) (*Policy, error) { + r, found, err := mongodb.NewPolicyColl().Get(ns, name) + if err != nil { + return nil, err + } else if !found { + log.Errorf("policy %s not found", name) + return nil, fmt.Errorf("policy %s not found", name) + } + res, err := buildPolicy(r) + if err != nil { + log.Errorf("buildPolicy error:%s", err) + return nil, err + } + return res, nil +} + +func DeletePolicy(name string, projectName string, logger *zap.SugaredLogger) error { + err := mongodb.NewPolicyColl().Delete(name, projectName) + if err != nil { + logger.Errorf("Failed to delete policy %s in project %s, err: %s", name, projectName, err) + return err + } + + return mongodb.NewPolicyBindingColl().DeleteByPolicy(name, projectName) +} + +func DeletePolicies(names []string, projectName string, logger *zap.SugaredLogger) error { + + err := mongodb.NewPolicyColl().DeleteMany(names, projectName) + if err != nil { + logger.Errorf("Failed to delete policies %s in project %s, err: %s", names, projectName, err) + return err + } + + return mongodb.NewPolicyBindingColl().DeleteByPolicies(names, projectName) +} + +func ListUserAllPoliciesByPolicyBindings(policyBindings []*models.PolicyBinding) ([]*models.Policy, error) { + var policies []*models.Policy + for _, v := range policyBindings { + tmpRoles, err := mongodb.NewPolicyColl().ListBySpaceAndName(v.PolicyRef.Namespace, v.PolicyRef.Name) + if err != nil { + continue + } + policies = append(policies, tmpRoles...) + } + return policies, nil +} diff --git a/pkg/microservice/policy/core/service/policy_binding.go b/pkg/microservice/policy/core/service/policy_binding.go new file mode 100644 index 0000000000000000000000000000000000000000..7a2f9c33dcf00e6d54f51959456533bfa26462e9 --- /dev/null +++ b/pkg/microservice/policy/core/service/policy_binding.go @@ -0,0 +1,148 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "fmt" + + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/policy/core/repository/models" + "github.com/koderover/zadig/pkg/microservice/policy/core/repository/mongodb" + "github.com/koderover/zadig/pkg/setting" +) + +type PolicyBinding struct { + Name string `json:"name"` + UID string `json:"uid"` + Policy string `json:"policy"` + Preset bool `json:"preset"` + Type setting.ResourceType `json:"type"` +} + +func CreatePolicyBindings(ns string, rbs []*PolicyBinding, logger *zap.SugaredLogger) error { + var objs []*models.PolicyBinding + for _, rb := range rbs { + obj, err := createPolicyBindingObject(ns, rb, logger) + if err != nil { + return err + } + + objs = append(objs, obj) + } + + return mongodb.NewPolicyBindingColl().BulkCreate(objs) +} + +func CreateOrUpdateSystemPolicyBinding(ns string, rb *PolicyBinding, logger *zap.SugaredLogger) error { + obj, err := createPolicyBindingObject(ns, rb, logger) + if err != nil { + return err + } + return mongodb.NewPolicyBindingColl().UpdateOrCreate(obj) +} + +func UpdateOrCreatePolicyBinding(ns string, rb *PolicyBinding, logger *zap.SugaredLogger) error { + obj, err := createPolicyBindingObject(ns, rb, logger) + if err != nil { + return err + } + return mongodb.NewPolicyBindingColl().UpdateOrCreate(obj) +} + +func ListPolicyBindings(ns, uid string, _ *zap.SugaredLogger) ([]*PolicyBinding, error) { + var policyBindings []*PolicyBinding + modelPolicyBindings, err := mongodb.NewPolicyBindingColl().ListBy(ns, uid) + if err != nil { + return nil, err + } + + for _, v := range modelPolicyBindings { + policyBindings = append(policyBindings, &PolicyBinding{ + Name: v.Name, + Policy: v.PolicyRef.Name, + UID: v.Subjects[0].UID, + Preset: v.PolicyRef.Namespace == "", + Type: v.Type, + }) + } + + return policyBindings, nil +} + +func ListPolicyBindingsByPolicy(ns, policyName string, publicPolicy bool, _ *zap.SugaredLogger) ([]*PolicyBinding, error) { + var policyBindings []*PolicyBinding + + policyNamespace := ns + if publicPolicy { + policyNamespace = "" + } + modelPolicyBindings, err := mongodb.NewPolicyBindingColl().List(&mongodb.ListPolicyOptions{PolicyName: policyName, PolicyNamespace: policyNamespace}) + if err != nil { + return nil, err + } + + for _, v := range modelPolicyBindings { + policyBindings = append(policyBindings, &PolicyBinding{ + Name: v.Name, + Policy: v.PolicyRef.Name, + UID: v.Subjects[0].UID, + Preset: v.PolicyRef.Namespace == "", + }) + } + + return policyBindings, nil +} + +func DeletePolicyBinding(name string, projectName string, _ *zap.SugaredLogger) error { + return mongodb.NewPolicyBindingColl().Delete(name, projectName) +} + +func DeletePolicyBindings(names []string, projectName string, userID string, _ *zap.SugaredLogger) error { + + if len(names) == 1 && names[0] == "*" { + names = []string{} + } + + return mongodb.NewPolicyBindingColl().DeleteMany(names, projectName, userID) +} + +func createPolicyBindingObject(ns string, rb *PolicyBinding, logger *zap.SugaredLogger) (*models.PolicyBinding, error) { + nsPolicy := ns + if rb.Preset { + nsPolicy = "" + } + policy, found, err := mongodb.NewPolicyColl().Get(nsPolicy, rb.Policy) + if err != nil { + logger.Errorf("Failed to get policy %s in namespace %s, err: %s", rb.Policy, nsPolicy, err) + return nil, err + } else if !found { + logger.Errorf("Policy %s is not found in namespace %s", rb.Policy, nsPolicy) + return nil, fmt.Errorf("policy %s not found", rb.Policy) + } + + return &models.PolicyBinding{ + Name: rb.Name, + Namespace: ns, + Subjects: []*models.Subject{{Kind: models.UserKind, UID: rb.UID}}, + PolicyRef: &models.PolicyRef{ + Name: policy.Name, + Namespace: policy.Namespace, + }, + Type: setting.ResourceTypeSystem, + }, nil +} diff --git a/pkg/microservice/policy/core/service/policy_meta_registration.go b/pkg/microservice/policy/core/service/policy_meta_registration.go new file mode 100644 index 0000000000000000000000000000000000000000..30f20f04d085cfe698a65398825c50b786fa4e46 --- /dev/null +++ b/pkg/microservice/policy/core/service/policy_meta_registration.go @@ -0,0 +1,155 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/koderover/zadig/pkg/microservice/policy/core/repository/models" + "github.com/koderover/zadig/pkg/microservice/policy/core/repository/mongodb" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/types" +) + +type PolicyDefinition struct { + Resource string `json:"resource"` + Alias string `json:"alias"` + Description string `json:"description"` + Rules []*PolicyRuleDefinition `json:"rules"` +} + +type PolicyRuleDefinition struct { + Action string `json:"action"` + Alias string `json:"alias"` + Description string `json:"description"` +} + +func CreateOrUpdatePolicyRegistration(p *types.PolicyMeta, _ *zap.SugaredLogger) error { + obj := &models.PolicyMeta{ + Resource: p.Resource, + Alias: p.Alias, + Description: p.Description, + } + + for _, r := range p.Rules { + rule := &models.PolicyMetaRule{ + Action: r.Action, + Alias: r.Alias, + Description: r.Description, + } + for _, ar := range r.Rules { + var as []models.Attribute + for _, a := range ar.MatchAttributes { + as = append(as, models.Attribute{Key: a.Key, Value: a.Value}) + } + + rule.Rules = append(rule.Rules, &models.ActionRule{ + Method: ar.Method, + Endpoint: ar.Endpoint, + ResourceType: ar.ResourceType, + IDRegex: ar.IDRegex, + MatchAttributes: as, + }) + } + obj.Rules = append(obj.Rules, rule) + } + + return mongodb.NewPolicyMetaColl().UpdateOrCreate(obj) +} + +func GetPolicyRegistrationDefinitions(scope, envType string, _ *zap.SugaredLogger) ([]*PolicyDefinition, error) { + policieMetas, err := mongodb.NewPolicyMetaColl().List() + if err != nil { + return nil, err + } + systemScopeSet := sets.NewString("TestCenter", "DataCenter", "Template", "DeliveryCenter") + projectScopeSet := sets.NewString("Workflow", "Environment", "ProductionEnvironment", "Test", "Delivery", "Build", "Service", "Scan") + systemPolicyMetas, projectPolicyMetas, filteredPolicyMetas := []*models.PolicyMeta{}, []*models.PolicyMeta{}, []*models.PolicyMeta{} + for _, v := range policieMetas { + if systemScopeSet.Has(v.Resource) { + systemPolicyMetas = append(systemPolicyMetas, v) + } else if projectScopeSet.Has(v.Resource) { + projectPolicyMetas = append(projectPolicyMetas, v) + } + } + + switch scope { + case string(types.SystemScope): + for _, meta := range systemPolicyMetas { + if meta.Resource == "TestCenter" { + tmpRules := []*models.PolicyMetaRule{} + for _, rule := range meta.Rules { + if rule.Action == "create_test" { + continue + } + tmpRules = append(tmpRules, rule) + } + + meta.Rules = tmpRules + } + } + filteredPolicyMetas = systemPolicyMetas + case string(types.ProjectScope): + tmp := []*models.PolicyMeta{} + for _, meta := range projectPolicyMetas { + if envType == setting.PMDeployType && (meta.Resource == "ProductionEnvironment" || meta.Resource == "Delivery") { + continue + } + if envType == setting.TrusteeshipDeployType && meta.Resource == "Delivery" { + continue + } + tmp = append(tmp, meta) + } + projectPolicyMetas = tmp + for i, meta := range projectPolicyMetas { + if meta.Resource == "Environment" || meta.Resource == "ProductionEnvironment" { + tmpRules := []*models.PolicyMetaRule{} + for _, rule := range meta.Rules { + if rule.Action == "debug_pod" && envType == setting.PMDeployType { + continue + } + if rule.Action == "ssh_pm" && envType != setting.PMDeployType { + continue + } + tmpRules = append(tmpRules, rule) + } + projectPolicyMetas[i].Rules = tmpRules + } + } + filteredPolicyMetas = projectPolicyMetas + default: + filteredPolicyMetas = policieMetas + } + var res []*PolicyDefinition + for _, meta := range filteredPolicyMetas { + pd := &PolicyDefinition{ + Resource: meta.Resource, + Alias: meta.Alias, + Description: meta.Description, + } + for _, r := range meta.Rules { + pd.Rules = append(pd.Rules, &PolicyRuleDefinition{ + Action: r.Action, + Alias: r.Alias, + Description: r.Description, + }) + } + res = append(res, pd) + } + return res, nil +} diff --git a/pkg/microservice/policy/core/service/role.go b/pkg/microservice/policy/core/service/role.go index 2e330ebe4df7ddaf610215087cca58cbc4e41d6e..ef5f1d77c00d35523786fc747dfa3cdd8e2e1211 100644 --- a/pkg/microservice/policy/core/service/role.go +++ b/pkg/microservice/policy/core/service/role.go @@ -17,34 +17,163 @@ limitations under the License. package service import ( + "fmt" + "go.uber.org/zap" "github.com/koderover/zadig/pkg/microservice/policy/core/repository/models" "github.com/koderover/zadig/pkg/microservice/policy/core/repository/mongodb" + "github.com/koderover/zadig/pkg/setting" ) type Role struct { - Name string `json:"name"` - Rules []*PolicyRule `json:"rules"` + Name string `json:"name"` + Rules []*Rule `json:"rules,omitempty"` + // if the role is created by user , set the type to "custom" + // if the role is created by system , set the type to "system" + Type setting.ResourceType `json:"type,omitempty"` + // frontend default select flag + Select bool `json:"select,omitempty"` + Namespace string `json:"namespace"` + Desc string `json:"desc,omitempty"` } -type PolicyRule struct { - Verbs []string `json:"verbs"` - Resources []string `json:"resources"` +func CreateRole(ns string, role *Role, _ *zap.SugaredLogger) error { + obj := &models.Role{ + Name: role.Name, + Namespace: ns, + Type: role.Type, + Desc: role.Desc, + } + + for _, r := range role.Rules { + obj.Rules = append(obj.Rules, &models.Rule{ + Verbs: r.Verbs, + Kind: r.Kind, + Resources: r.Resources, + MatchAttributes: r.MatchAttributes, + }) + } + + return mongodb.NewRoleColl().Create(obj) } -func CreateRole(ns string, role *Role, _ *zap.SugaredLogger) error { +func UpdateRole(ns string, role *Role, _ *zap.SugaredLogger) error { obj := &models.Role{ Name: role.Name, Namespace: ns, } for _, r := range role.Rules { - obj.Rules = append(obj.Rules, &models.PolicyRule{ + obj.Rules = append(obj.Rules, &models.Rule{ Verbs: r.Verbs, + Kind: r.Kind, Resources: r.Resources, }) } + return mongodb.NewRoleColl().UpdateRole(obj) +} - return mongodb.NewRoleColl().Create(obj) +func UpdateOrCreateRole(ns string, role *Role, _ *zap.SugaredLogger) error { + obj := &models.Role{ + Name: role.Name, + Desc: role.Desc, + Namespace: ns, + Type: role.Type, + } + + for _, r := range role.Rules { + obj.Rules = append(obj.Rules, &models.Rule{ + Verbs: r.Verbs, + Kind: r.Kind, + Resources: r.Resources, + MatchAttributes: r.MatchAttributes, + }) + } + return mongodb.NewRoleColl().UpdateOrCreate(obj) +} + +func ListRoles(projectName string, _ *zap.SugaredLogger) ([]*Role, error) { + var roles []*Role + projectRoles, err := mongodb.NewRoleColl().ListBy(projectName) + if err != nil { + return nil, err + } + for _, v := range projectRoles { + // frontend doesn't need to see contributor role + if v.Name == string(setting.Contributor) { + continue + } + tmpRole := Role{Select: false, Name: v.Name, Type: v.Type, Desc: v.Desc, Namespace: v.Namespace} + if v.Name == string(setting.ReadProjectOnly) { + tmpRole.Select = true + } + roles = append(roles, &tmpRole) + } + return roles, nil +} + +func GetRole(ns, name string, _ *zap.SugaredLogger) (*Role, error) { + r, found, err := mongodb.NewRoleColl().Get(ns, name) + if err != nil { + return nil, err + } else if !found { + return nil, fmt.Errorf("role %s not found", name) + } + + res := &Role{ + Name: r.Name, + } + for _, ru := range r.Rules { + res.Rules = append(res.Rules, &Rule{ + Verbs: ru.Verbs, + Kind: ru.Kind, + Resources: ru.Resources, + }) + } + + return res, nil +} + +func DeleteRole(name string, projectName string, logger *zap.SugaredLogger) error { + err := mongodb.NewRoleColl().Delete(name, projectName) + if err != nil { + logger.Errorf("Failed to delete role %s in project %s, err: %s", name, projectName, err) + return err + } + + return mongodb.NewRoleBindingColl().DeleteByRole(name, projectName) +} + +func DeleteRoles(names []string, projectName string, logger *zap.SugaredLogger) error { + if len(names) == 0 { + return nil + } + if projectName == "" { + return fmt.Errorf("projectName is empty") + } + + if names[0] == "*" { + names = []string{} + } + + err := mongodb.NewRoleColl().DeleteMany(names, projectName) + if err != nil { + logger.Errorf("Failed to delete roles %s in project %s, err: %s", names, projectName, err) + return err + } + + return mongodb.NewRoleBindingColl().DeleteByRoles(names, projectName) +} + +func ListUserAllRolesByRoleBindings(roleBindings []*models.RoleBinding) ([]*models.Role, error) { + var roles []*models.Role + for _, v := range roleBindings { + tmpRoles, err := mongodb.NewRoleColl().ListBySpaceAndName(v.RoleRef.Namespace, v.RoleRef.Name) + if err != nil { + continue + } + roles = append(roles, tmpRoles...) + } + return roles, nil } diff --git a/pkg/microservice/policy/core/service/role_binding.go b/pkg/microservice/policy/core/service/role_binding.go index 8300729c59e44164d10df0d2ee64ddc46a5c9f6c..c50630324d861198ee3fbc38d58d40227fa15e4e 100644 --- a/pkg/microservice/policy/core/service/role_binding.go +++ b/pkg/microservice/policy/core/service/role_binding.go @@ -21,45 +21,178 @@ import ( "go.uber.org/zap" + "github.com/koderover/zadig/pkg/config" "github.com/koderover/zadig/pkg/microservice/policy/core/repository/models" "github.com/koderover/zadig/pkg/microservice/policy/core/repository/mongodb" + "github.com/koderover/zadig/pkg/setting" ) type RoleBinding struct { - Name string `json:"name"` - User string `json:"user"` - Role string `json:"role"` - Global bool `json:"global"` + Name string `json:"name"` + UID string `json:"uid"` + Role string `json:"role"` + Preset bool `json:"preset"` + Type setting.ResourceType `json:"type"` } -func CreateRoleBinding(ns string, rb *RoleBinding, logger *zap.SugaredLogger) error { - if ns == "" { - logger.Errorf("Namespace is empty") - return fmt.Errorf("empty namespace") +func CreateRoleBindings(ns string, rbs []*RoleBinding, logger *zap.SugaredLogger) error { + var objs []*models.RoleBinding + for _, rb := range rbs { + obj, err := createRoleBindingObject(ns, rb, logger) + if err != nil { + return err + } + + objs = append(objs, obj) + } + + return mongodb.NewRoleBindingColl().BulkCreate(objs) +} + +func UpdateRoleBindings(ns string, rbs []*RoleBinding, userID string, logger *zap.SugaredLogger) error { + err := DeleteRoleBindings([]string{"*"}, ns, userID, logger) + if err != nil { + logger.Errorf("delete rolebindings err:%s,ns:%s,userID:%s", err, ns, userID) + return err + } + err = CreateRoleBindings(ns, rbs, logger) + if err != nil { + logger.Errorf("create rolebings err:%s,ns:%s,userID:%s", err, ns, userID) + return err + } + return nil +} + +func CreateOrUpdateSystemRoleBinding(ns string, rb *RoleBinding, logger *zap.SugaredLogger) error { + + obj, err := createRoleBindingObject(ns, rb, logger) + if err != nil { + return err + } + return mongodb.NewRoleBindingColl().UpdateOrCreate(obj) +} + +func UpdateOrCreateRoleBinding(ns string, rb *RoleBinding, logger *zap.SugaredLogger) error { + obj, err := createRoleBindingObject(ns, rb, logger) + if err != nil { + return err } + return mongodb.NewRoleBindingColl().UpdateOrCreate(obj) +} +func ListRoleBindings(ns, uid string, _ *zap.SugaredLogger) ([]*RoleBinding, error) { + var roleBindings []*RoleBinding + modelRoleBindings, err := mongodb.NewRoleBindingColl().ListBy(ns, uid) + if err != nil { + return nil, err + } + + for _, v := range modelRoleBindings { + roleBindings = append(roleBindings, &RoleBinding{ + Name: v.Name, + Role: v.RoleRef.Name, + UID: v.Subjects[0].UID, + Preset: v.RoleRef.Namespace == "", + }) + } + + return roleBindings, nil +} + +func SearchSystemRoleBindings(uids []string, _ *zap.SugaredLogger) (map[string][]*RoleBinding, error) { + var roleBindings []*RoleBinding + modelRoleBindings, err := mongodb.NewRoleBindingColl().ListSystemRoleBindingsByUIDs(uids) + if err != nil { + return nil, err + } + + for _, v := range modelRoleBindings { + roleBindings = append(roleBindings, &RoleBinding{ + Name: v.Name, + Role: v.RoleRef.Name, + UID: v.Subjects[0].UID, + Preset: v.RoleRef.Namespace == "", + }) + } + resMap := make(map[string][]*RoleBinding) + for _, rb := range roleBindings { + resMap[rb.UID] = append(resMap[rb.UID], rb) + } + + return resMap, nil +} + +func DeleteRoleBinding(name string, projectName string, _ *zap.SugaredLogger) error { + return mongodb.NewRoleBindingColl().Delete(name, projectName) +} + +func DeleteRoleBindings(names []string, projectName string, userID string, _ *zap.SugaredLogger) error { + + if len(names) == 1 && names[0] == "*" { + names = []string{} + } + + return mongodb.NewRoleBindingColl().DeleteMany(names, projectName, userID) +} + +func createRoleBindingObject(ns string, rb *RoleBinding, logger *zap.SugaredLogger) (*models.RoleBinding, error) { nsRole := ns - if rb.Global { + if rb.Preset { nsRole = "" } role, found, err := mongodb.NewRoleColl().Get(nsRole, rb.Role) if err != nil { logger.Errorf("Failed to get role %s in namespace %s, err: %s", rb.Role, nsRole, err) - return err + return nil, err } else if !found { logger.Errorf("Role %s is not found in namespace %s", rb.Role, nsRole) - return fmt.Errorf("role %s not found", rb.Role) + return nil, fmt.Errorf("role %s not found", rb.Role) } - obj := &models.RoleBinding{ + ensureRoleBindingName(ns, rb) + + return &models.RoleBinding{ Name: rb.Name, Namespace: ns, - Subjects: []*models.Subject{{Kind: models.UserKind, Name: rb.User}}, + Subjects: []*models.Subject{{Kind: models.UserKind, UID: rb.UID}}, RoleRef: &models.RoleRef{ Name: role.Name, Namespace: role.Namespace, }, + }, nil +} + +func ensureRoleBindingName(ns string, rb *RoleBinding) { + if rb.Name != "" { + return + } + + nsRole := ns + if rb.Preset { + nsRole = "" } - return mongodb.NewRoleBindingColl().Create(obj) + rb.Name = config.RoleBindingNameFromUIDAndRole(rb.UID, setting.RoleType(rb.Role), nsRole) +} + +func ListUserAllRoleBindings(projectName, uid string) ([]*models.RoleBinding, error) { + var rbs []mongodb.RoleBinding + roleBindingReadOnly := mongodb.RoleBinding{ + Uid: "*", + Namespace: projectName, + } + roleBindingsAdmin := mongodb.RoleBinding{ + Uid: uid, + Namespace: "*", + } + roleBindingCommon := mongodb.RoleBinding{ + Uid: uid, + Namespace: projectName, + } + rbs = append(rbs, roleBindingReadOnly, roleBindingsAdmin, roleBindingCommon) + roleBindings, err := mongodb.NewRoleBindingColl().ListByRoleBindingOpt(mongodb.ListRoleBindingsOpt{RoleBindings: rbs}) + if err != nil { + return nil, err + } + return roleBindings, nil } diff --git a/pkg/microservice/policy/core/yamlconfig/meta.go b/pkg/microservice/policy/core/yamlconfig/meta.go new file mode 100644 index 0000000000000000000000000000000000000000..13084ed485d544dbda3b0ef298afa43d6d5254f3 --- /dev/null +++ b/pkg/microservice/policy/core/yamlconfig/meta.go @@ -0,0 +1,169 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package yamlconfig + +import ( + _ "embed" + "strings" + + "sigs.k8s.io/yaml" + + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types" + "github.com/koderover/zadig/pkg/util/deepcopy" +) + +//go:embed metas.yaml +var PolicyMetas []byte + +var defaultMetaConfig *MetaConfig + +func init() { + log.Init(&log.Config{ + Level: "debug", + NoCaller: true, + }) + defaultMetaConfig = newMetaConfigFromEmbed() +} + +func DefaultPolicyMetas() []*types.PolicyMeta { + return defaultMetaConfig.Policies() +} + +func DefaultPolicyMetasConfig() *MetaConfig { + return defaultMetaConfig +} + +func PolicyMetasFromBytes(bs []byte) []*types.PolicyMeta { + return newMetaConfigFromBytes(bs).Policies() +} + +type MetaGetter interface { + Policies() []*types.PolicyMeta +} + +type MetaConfig struct { + Description string `json:"description"` + Metas []*types.PolicyMeta `json:"metas"` +} + +func (m *MetaConfig) Policies() []*types.PolicyMeta { + m.Metas = processMetas(m.Metas) + return m.Metas +} + +func newMetaConfigFromBytes(b []byte) MetaGetter { + m := &MetaConfig{} + if err := yaml.Unmarshal(b, &m); err != nil { + log.Warnf("fail to new meta getter err:%s, use the default meta getter", err) + return defaultMetaConfig + } + return m +} + +func newMetaConfigFromEmbed() *MetaConfig { + config := &MetaConfig{} + if err := yaml.Unmarshal(PolicyMetas, &config); err != nil { + log.DPanic(err) + } + return config +} + +func processMetas(metas []*types.PolicyMeta) []*types.PolicyMeta { + proEnvMeta := &types.PolicyMeta{} + + for _, meta := range metas { + if meta.Resource == "Workflow" { + for _, ruleMeta := range meta.Rules { + tmpRules := []*types.ActionRule{} + for _, rule := range ruleMeta.Rules { + if rule.ResourceType == "" { + rule.ResourceType = "Workflow" + } + if strings.Contains(rule.Endpoint, ":name") { + idRegex := strings.ReplaceAll(rule.Endpoint, ":name", `([\w\W].*)`) + idRegex = strings.ReplaceAll(idRegex, "?*", `[\w\W].*`) + endpoint := strings.ReplaceAll(rule.Endpoint, ":name", "?*") + rule.Endpoint = endpoint + rule.IDRegex = idRegex + } + if rule.Filter { + rule.MatchAttributes = []*types.Attribute{ + { + Key: "placeholder", + Value: "placeholder", + }, + } + } + tmpRules = append(tmpRules, rule) + } + ruleMeta.Rules = tmpRules + } + } + if meta.Resource == "Environment" { + for _, ruleMeta := range meta.Rules { + tmpRules := []*types.ActionRule{} + for _, rule := range ruleMeta.Rules { + if rule.ResourceType == "" { + rule.ResourceType = "Environment" + } + if rule.Filter { + rule.MatchAttributes = []*types.Attribute{ + { + Key: "production", + Value: "false", + }, + } + } + if strings.Contains(rule.Endpoint, ":name") { + idRegex := strings.ReplaceAll(rule.Endpoint, ":name", `([\w\W].*)`) + idRegex = strings.ReplaceAll(idRegex, "?*", `[\w\W].*`) + endpoint := strings.ReplaceAll(rule.Endpoint, ":name", "?*") + rule.Endpoint = endpoint + rule.IDRegex = idRegex + rule.MatchAttributes = []*types.Attribute{ + { + Key: "production", + Value: "false", + }, + } + } + + tmpRules = append(tmpRules, rule) + } + ruleMeta.Rules = tmpRules + } + + if err := deepcopy.FromTo(meta, proEnvMeta); err != nil { + log.DPanic(err) + } + proEnvMeta.Resource = "ProductionEnvironment" + proEnvMeta.Alias = "环境(生产/预发布)" + for _, ru := range proEnvMeta.Rules { + for _, r := range ru.Rules { + for _, a := range r.MatchAttributes { + if a.Key == "production" { + a.Value = "true" + } + } + } + } + } + } + metas = append(metas, proEnvMeta) + return metas +} diff --git a/pkg/microservice/policy/core/yamlconfig/metas.yaml b/pkg/microservice/policy/core/yamlconfig/metas.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d64ebea5eb790fb115e0fbf5ec874bfedace5539 --- /dev/null +++ b/pkg/microservice/policy/core/yamlconfig/metas.yaml @@ -0,0 +1,773 @@ +description: " + public urls: url not need to authentication ; system_admin urls: urls that + only system admin has permission ; project_admin: urls that project admin has + permission" +metas: + - resource: Delivery + alias: 版本管理 + description: '' + rules: + - action: get_delivery + alias: 查看 + description: '' + rules: + - method: GET + endpoint: /api/aslan/delivery/releases/helm/charts + - method: GET + endpoint: /api/aslan/delivery/releases + - action: delete_delivery + alias: 删除 + description: '' + rules: + - method: DELETE + endpoint: /api/aslan/delivery/releases/?* + - action: create_delivery + alias: 创建 + description: '' + rules: + - method: POST + endpoint: /api/aslan/delivery/releases/helm + - method: GET + endpoint: /api/aslan/environment/environments + - method: GET + endpoint: /api/aslan/environment/environments/?*/helm/releases + - method: GET + endpoint: /api/aslan/environment/environments/?*/helm/charts + - method: GET + endpoint: /api/aslan/service/helm/?*/?*/fileContent + - method: POST + endpoint: /api/aslan/delivery/releases/helm/global-variables + - method: GET + endpoint: /api/aslan/delivery/releases/helm/charts/version + - resource: Test + alias: 测试 + description: '' + rules: + - action: get_test + alias: 查看 + description: '' + rules: + - method: GET + endpoint: /api/aslan/workflow/sse/workflows/id/?*/pipelines/?* + - method: GET + endpoint: /api/aslan/workflow/workflowtask/id/?*/pipelines/?* + - method: GET + endpoint: /api/aslan/workflow/workflowtask/max/?*/start/?*/pipelines/?* + - method: GET + endpoint: /api/aslan/testing/test + - method: GET + endpoint: /api/aslan/testing/test/?* + - method: GET + endpoint: /api/aslan/testing/testdetail + - method: GET + endpoint: /api/aslan/workflow/workflow/testName/?* + - method: GET + endpoint: /api/aslan/workflow/v2/tasks/workflow/workflow/?*/taskId/?* + - method: GET + endpoint: /api/aslan/testing/workspace/workflow/?*/taskId/?* + - method: GET + endpoint: /api/aslan/cluster/clusters + - action: edit_test + alias: 编辑 + description: '' + rules: + - method: PUT + endpoint: /api/aslan/testing/test + - action: delete_test + alias: 删除 + description: '' + rules: + - method: DELETE + endpoint: /api/aslan/testing/test/?* + - action: create_test + alias: 新建 + description: '' + rules: + - method: POST + endpoint: /api/aslan/testing/test + - action: run_test + alias: 执行 + description: '' + rules: + - method: DELETE + endpoint: api/aslan/testing/testtask/productName/?*/id/?*/pipelines/?* + - method: POST + endpoint: /api/aslan/testing/testtask + - method: POST + endpoint: /api/aslan/testing/testtask/productName/?*/id/?*/pipelines/?*/restart + - resource: Service + alias: 服务 + description: '' + rules: + - action: get_service + alias: 查看 + description: '' + rules: + - method: GET + endpoint: api/aslan/build/targets + - method: GET + endpoint: api/aslan/cluster/clusters + - method: GET + endpoint: api/aslan/build/build/?* + - method: GET + endpoint: api/aslan/build/build + - method: GET + endpoint: /api/aslan/template/build + - method: GET + endpoint: api/aslan/template/dockerfile + - method: GET + endpoint: /api/aslan/service/services + - method: GET + endpoint: /api/aslan/service/services/?*/k8s + - method: GET + endpoint: /api/aslan/project/products/?*/services + - method: GET + endpoint: /api/aslan/service/services/?* + - method: GET + endpoint: /api/aslan/project/products/?*/searching-rules + - method: GET + endpoint: /api/aslan/service/helm/?*/?*/filePath + - method: GET + endpoint: /api/aslan/service/helm/?*/?*/fileContent + - method: GET + endpoint: /api/aslan/service/helm/?*/?*/serviceModule + - method: GET + endpoint: /api/aslan/service/services/?*/pm + - method: GET + endpoint: /api/aslan/template/yaml + - method: GET + endpoint: /api/aslan/service/helm/?*/?*/fileContent + - action: edit_service + alias: 编辑 + description: '' + rules: + - method: POST + endpoint: /api/aslan/service/helm/?* + - method: PUT + endpoint: /api/aslan/service/services + - method: POST + endpoint: /api/aslan/service/services + - method: PUT + endpoint: /api/aslan/service/pm/?* + - method: PUT + endpoint: /api/aslan/project/products/?* + - method: PATCH + endpoint: /api/aslan/project/products/?* + - method: PUT + endpoint: /api/aslan/project/products/?*/searching-rules + - method: PUT + endpoint: /api/aslan/service/helm/services/releaseNaming + - action: create_service + alias: 新建 + description: '' + rules: + - method: POST + endpoint: /api/aslan/service/pm/?* + - method: POST + endpoint: /api/aslan/service/services + - method: POST + endpoint: /api/aslan/service/loader/load/?*/?* + - method: PUT + endpoint: /api/aslan/service/loader/load/?*/?* + - method: POST + endpoint: /api/aslan/service/template/load + - method: POST + endpoint: /api/aslan/service/helm/services + - method: PUT + endpoint: /api/aslan/service/helm/services + - method: POST + endpoint: /api/aslan/service/helm/services/bulk + - method: GET + endpoint: /api/aslan/service/services/kube/workloads + - method: POST + endpoint: /api/aslan/service/services/yaml + - action: delete_service + alias: 删除 + description: '' + rules: + - method: DELETE + endpoint: /api/aslan/service/services/?*/?* + - method: DELETE + endpoint: /api/aslan/service/services/?*/pm + - resource: Build + alias: 构建 + description: '' + rules: + - action: get_build + alias: 查看 + description: '' + rules: + - method: GET + endpoint: /api/aslan/build/build + - method: GET + endpoint: /api/aslan/build/build/?* + - method: GET + endpoint: /api/aslan/build/targets + - method: GET + endpoint: /api/aslan/cluster/clusters + - action: create_build + alias: 新建 + description: '' + rules: + - method: GET + endpoint: api/aslan/template/build + - method: GET + endpoint: api/aslan/template/dockerfile + - method: POST + endpoint: /api/aslan/build/build + - action: edit_build + alias: 编辑 + description: '' + rules: + - method: PUT + endpoint: /api/aslan/build/build + - method: POST + endpoint: /api/aslan/build/targets + - method: POST + endpoint: /api/aslan/build/build/targets + - action: delete_build + alias: 删除 + description: '' + rules: + - method: DELETE + endpoint: /api/aslan/build/build + - resource: TestCenter + alias: 测试中心 + description: '' + rules: + - action: get_test + alias: 查看 + description: '' + rules: + - method: GET + endpoint: /api/aslan/workflow/sse/workflows/id/?*/pipelines/?* + - method: GET + endpoint: /api/aslan/testing/test + - method: GET + endpoint: /api/aslan/testing/test/?* + - method: GET + endpoint: /api/aslan/testing/testdetail + - method: GET + endpoint: /api/aslan/workflow/workflow/testName/?* + - method: GET + endpoint: /api/aslan/workflow/v2/tasks/workflow/workflow/?*/taskId/?* + - method: GET + endpoint: /api/aslan/testing/workspace/workflow/?*/taskId/?* + - action: edit_test + alias: 编辑 + description: '' + rules: + - method: PUT + endpoint: /api/aslan/testing/test + - method: GET + endpoint: /api/aslan/cluster/clusters + - action: delete_test + alias: 删除 + description: '' + rules: + - method: DELETE + endpoint: /api/aslan/testing/test/?* + - action: create_test + alias: 新建 + description: '' + rules: + - method: POST + endpoint: /api/aslan/testing/test + - method: GET + endpoint: /api/aslan/cluster/clusters + - action: run_test + alias: 执行 + description: '' + rules: + - method: POST + endpoint: /api/aslan/testing/testtask + - resource: DataCenter + alias: 数据视图 + description: '' + rules: + - action: data_over + alias: 数据概览|查看 + description: '' + rules: + - method: GET + endpoint: /api/v1/picket/stat/dashboard/overview + - method: GET + endpoint: /api/v1/picket/stat/dashboard/test + - method: GET + endpoint: api/v1/picket/stat/dashboard/deploy + - method: GET + endpoint: /api/v1/picket/stat/dashboard/build + - action: efficiency_over + alias: 数据洞察|查看 + description: '' + rules: + - method: GET + endpoint: /api/aslan/stat/dashboard/build + - method: GET + endpoint: /api/aslan/stat/dashboard/deploy + - method: GET + endpoint: /api/aslan/stat/dashboard/test + - method: POST + endpoint: /api/aslan/stat/quality/buildHealthMeasure + - method: POST + endpoint: /api/aslan/stat/quality/buildLatestTenMeasure + - method: POST + endpoint: /api/aslan/stat/quality/buildDailyMeasure + - method: POST + endpoint: /api/aslan/stat/quality/buildTenDurationMeasure + - method: POST + endpoint: /api/aslan/stat/quality/buildAverageMeasure + - method: POST + endpoint: api/aslan/stat/quality/buildTrend + - method: POST + endpoint: api/aslan/stat/quality/deployTopFiveFailureMeasure + - method: POST + endpoint: api/aslan/stat/quality/deployTopFiveHigherMeasure + - method: POST + endpoint: api/aslan/stat/quality/deployWeeklyMeasure + - method: POST + endpoint: api/aslan/stat/quality/deployHealthMeasure + - method: POST + endpoint: api/aslan/stat/quality/testDeliveryDeploy + - method: POST + endpoint: api/aslan/stat/quality/testAverageMeasure + - method: POST + endpoint: api/aslan/stat/quality/testCaseMeasure + - method: POST + endpoint: api/aslan/stat/quality/testHealthMeasure + - method: POST + endpoint: api/aslan/stat/quality/testTrend + - resource: Template + alias: 模版库 + description: '' + rules: + - action: create_template + alias: 新建 + description: 新建 + rules: + - method: POST + endpoint: /api/aslan/template/dockerfile + - method: POST + endpoint: /api/aslan/template/charts + - method: POST + endpoint: /api/aslan/template/yaml + - method: POST + endpoint: /api/aslan/template/build + - action: get_template + alias: 查看 + description: 查看 + rules: + - method: GET + endpoint: /api/aslan/cluster/clusters + - method: GET + endpoint: /api/aslan/template/yaml + - method: GET + endpoint: /api/aslan/template/charts + - method: GET + endpoint: /api/aslan/template/charts/?* + - method: GET + endpoint: /api/aslan/template/dockerfile + - method: GET + endpoint: /api/aslan/template/build + - method: GET + endpoint: /api/aslan/template/build/?* + - action: edit_template + alias: 编辑 + description: 编辑 + rules: + - method: PUT + endpoint: /api/aslan/template/yaml/?* + - method: PUT + endpoint: /api/aslan/template/charts/?* + - method: POST + endpoint: api/aslan/template/charts/?*/reference + - method: POST + endpoint: api/aslan/template/yaml/?*/reference + - method: PUT + endpoint: api/aslan/template/charts/?*/variables + - method: PUT + endpoint: /api/aslan/template/dockerfile/?* + - method: PUT + endpoint: /api/aslan/template/build/?* + - action: delete_template + alias: 删除 + description: 删除 + rules: + - method: DELETE + endpoint: /api/aslan/template/yaml/?* + - method: DELETE + endpoint: /api/aslan/template/charts/?* + - method: DELETE + endpoint: /api/aslan/template/dockerfile/?* + - method: DELETE + endpoint: /api/aslan/template/build/?* + - resource: DeliveryCenter + alias: 交付中心 + description: '' + rules: + - action: release_get + alias: 版本管理|查看 + description: 查看 + rules: + - method: GET + endpoint: /api/aslan/delivery/releases + - method: GET + endpoint: /api/aslan/delivery/releases/?* + - action: delivery_get + alias: 交付物追踪|查看 + description: 查看 + rules: + - method: GET + endpoint: /api/aslan/delivery/artifacts + - resource: Workflow + alias: 工作流 + description: '' + rules: + - action: get_workflow + alias: 查看 + description: '' + rules: + - method: GET + endpoint: /api/aslan/workflow/workflow/preset/?* + - method: GET + endpoint: api/aslan/workflow/workflow + filter: true + - method: GET + endpoint: api/aslan/workflow/workflow + - method: GET + endpoint: /api/aslan/workflow/workflow/find/?* + - method: GET + endpoint: /api/aslan/workflow/workflowtask/max/?*/start/?*/pipelines/?* + - method: GET + endpoint: /api/aslan/workflow/workflowtask/id/?*/pipelines/?* + - method: GET + endpoint: /api/aslan/workflow/sse/workflows/id/?*/pipelines/?* + - method: GET + endpoint: /api/aslan/logs/sse/workflow/build/?*/?*/?*/?* + - method: GET + endpoint: /api/aslan/logs/log/workflow/?*/tasks/?*/service/?* + - method: GET + endpoint: /api/aslan/logs/sse/workflow/test/?*/?*/?*/?*/?* + - method: GET + endpoint: /api/aslan/logs/log/workflow/?*/tasks/?*/tests/test/service/?* + - method: GET + endpoint: /api/aslan/logs/log/workflow/?*/tasks/?*/tests/test/service/?* + - method: GET + endpoint: /api/aslan/testing/itreport/workflow/?*/id/?*/names/?*/service/?* + - method: GET + endpoint: /api/directory/workflowTask + - method: GET + endpoint: /api/aslan/workflow/v3 + - method: GET + endpoint: /api/aslan/workflow/v3/?* + - method: GET + endpoint: /api/aslan/workflow/v3/?*/args + - method: GET + endpoint: /api/aslan/workflow/servicetask/workflows/?*/?*/?*/?* + - method: GET + endpoint: api/aslan/workflow/v4 + filter: true + - method: GET + endpoint: api/aslan/workflow/v4 + - method: GET + endpoint: /api/aslan/workflow/v4/name/?* + - method: GET + endpoint: /api/aslan/workflow/v4/preset/?* + - method: GET + endpoint: /api/aslan/workflow/v4/workflowtask + - method: GET + endpoint: /api/aslan/workflow/v4/workflowtask/workflow/?*/task/?* + - method: GET + endpoint: /api/aslan/workflow/v4/workflowtask/clone/workflow/?*/task/?* + - method: GET + endpoint: /api/aslan/logs/log/v4/workflow/?*/tasks/?*/jobs/?* + - method: GET + endpoint: /api/aslan/logs/sse/v4/workflow/?*/tasks/?*/jobs/?* + - action: edit_workflow + alias: 编辑 + description: '' + rules: + - method: PUT + endpoint: 'api/aslan/workflow/workflow/:name' + - method: GET + endpoint: /api/aslan/environment/environments + resourceType: Environment + filter: true + - method: GET + endpoint: /api/aslan/testing/testdetail + - method: PUT + endpoint: /api/aslan/workflow/v3/?* + - method: GET + endpoint: /api/aslan/build/build/serviceModule + - method: PUT + endpoint: /api/aslan/workflow/v4/?* + - method: POST + endpoint: /api/aslan/workflow/v4/lint + - action: create_workflow + alias: 新建 + description: '' + rules: + - method: GET + endpoint: /api/aslan/build/build/serviceModule + - method: POST + endpoint: api/aslan/workflow/workflow + - method: PUT + endpoint: /api/aslan/workflow/workflow/old/?*/new/?* + - method: POST + endpoint: /api/directory/workflowTask/create + - method: GET + endpoint: /api/aslan/environment/environments + resourceType: Environment + filter: true + - method: GET + endpoint: /api/aslan/testing/testdetail + - method: POST + endpoint: /api/aslan/workflow/v3 + - method: POST + endpoint: /api/aslan/workflow/v4 + - method: POST + endpoint: /api/aslan/workflow/v4/lint + - action: delete_workflow + alias: 删除 + description: '' + rules: + - method: DELETE + endpoint: /api/aslan/workflow/workflow/?* + - method: DELETE + endpoint: /api/aslan/workflow/v3/?* + - method: DELETE + endpoint: /api/aslan/workflow/v4/?* + - action: run_workflow + alias: 执行 + description: '' + rules: + - method: POST + endpoint: '/api/aslan/workflow/workflowtask/:name' + - method: PUT + endpoint: '/api/aslan/workflow/workflowtask/:name' + - method: POST + endpoint: '/api/aslan/workflow/workflowtask/id/?*/pipelines/:name/restart' + - method: DELETE + endpoint: '/api/aslan/workflow/workflowtask/id/?*/pipelines/:name' + - method: POST + endpoint: /api/directory/workflowTask/id/?*/pipelines/?*/restart + - method: POST + endpoint: /api/directory/workflowTask/id/?*/pipelines/?*/cancel + - method: GET + endpoint: /api/aslan/environment/environments + resourceType: Environment + filter: true + - method: GET + endpoint: /api/aslan/project/products/?*/services + - method: GET + endpoint: /api/aslan/workflow/v3/?*/args + - method: POST + endpoint: /api/aslan/workflow/v3/workflowtask + - method: POST + endpoint: /api/aslan/workflow/v3/workflowtask/id/?*/name/?*/restart + - method: DELETE + endpoint: /api/aslan/workflow/v3/workflowtask/id/?*/name/?* + - method: POST + endpoint: /api/aslan/workflow/v4/workflowtask + - method: DELETE + endpoint: /api/aslan/workflow/v4/workflowtask/workflow/?*/task/?* + - method: POST + endpoint: /api/aslan/workflow/v4/workflowtask/approve + - resource: Environment + alias: 环境 + description: '' + rules: + - action: get_environment + alias: 查看 + description: '' + rules: + - method: GET + endpoint: '/api/aslan/environment/environments/:name/helm/values' + - method: GET + endpoint: /api/aslan/environment/environments + - method: GET + endpoint: '/api/aslan/environment/environments/:name' + - method: GET + endpoint: '/api/aslan/environment/environments/:name/groups' + - method: GET + endpoint: '/api/aslan/environment/environments/:name/services/?*' + - method: GET + endpoint: /api/aslan/environment/kube/workloads + - method: GET + endpoint: /api/aslan/service/workloads + - method: GET + endpoint: /api/aslan/environment/export/service + - method: GET + endpoint: /api/aslan/environment/configmaps + - method: GET + endpoint: /api/aslan/environment/kube/pods/?*/events + - method: GET + endpoint: /api/aslan/environment/kube/events + - method: GET + endpoint: /api/aslan/logs/sse/pods/?*/containers/?* + - method: GET + endpoint: /api/aslan/project/products/?*/services + - method: GET + endpoint: /api/aslan/environment/revision/products + - method: GET + endpoint: '/api/aslan/environment/environments/:name/workloads' + - method: GET + endpoint: '/api/aslan/environment/environments/:name/helm/releases' + - method: GET + endpoint: '/api/aslan/environment/environments/:name/helm/charts' + - method: GET + endpoint: '/api/aslan/environment/environments/:name/helm/images' + - method: GET + endpoint: /api/aslan/environment/diff/products/?*/service/?* + - method: GET + endpoint: '/api/aslan/environment/configmaps/:name' + - method: GET + endpoint: '/api/aslan/environment/envcfgs/:name/cfg/?*' + - method: GET + endpoint: '/api/aslan/environment/secrets/:name' + - method: GET + endpoint: '/api/aslan/environment/ingresses/:name' + - method: GET + endpoint: '/api/aslan/environment/pvcs/:name' + - action: create_environment + alias: 创建 + description: '' + rules: + - method: GET + endpoint: api/aslan/delivery/releases + - method: POST + endpoint: /api/aslan/environment/environments + resourceType: Cluster + filter: true + - method: POST + endpoint: /api/aslan/service/workloads + - method: GET + endpoint: /api/aslan/project/products/?*/services + - method: GET + endpoint: api/aslan/cluster/clusters + - action: config_environment + alias: 配置 + description: '' + rules: + - method: PUT + endpoint: '/api/aslan/environment/environments/:name/helm/default-values' + - method: PUT + endpoint: '/api/aslan/environment/environments/:name/helm/charts' + - method: GET + endpoint: /api/aslan/build/targets + - method: GET + endpoint: /api/aslan/build/build + - method: GET + endpoint: /api/aslan/service/services/?* + - method: PUT + endpoint: '/api/aslan/environment/environments/:name/services' + - method: POST + endpoint: '/api/aslan/environment/environments/:name/share/enable' + - method: DELETE + endpoint: '/api/aslan/environment/environments/:name/share/enable' + - method: PUT + endpoint: /api/aslan/environment/environments + - method: PUT + endpoint: '/api/aslan/environment/environments/:name' + - method: PUT + endpoint: /api/aslan/service/pm/healthCheckUpdate + - method: PUT + endpoint: '/api/aslan/environment/environments/:name/envRecycle' + - method: PUT + endpoint: '/api/aslan/environment/environments/:name/renderset' + - method: PUT + endpoint: /api/aslan/service/workloads + - method: GET + endpoint: /api/aslan/project/products/?*/services + - method: PUT + endpoint: '/api/aslan/environment/environments/:name/registry' + - method: PUT + endpoint: '/api/aslan/environment/envcfgs/:name' + - method: POST + endpoint: '/api/aslan/environment/envcfgs/:name' + - method: DELETE + endpoint: '/api/aslan/environment/envcfgs/:name/cfg/?*' + - action: manage_environment + alias: 管理服务实例 + description: '' + rules: + - method: PUT + endpoint: /api/aslan/environment/environments + - method: POST + endpoint: '/api/aslan/environment/environments/:name/services/?*/restart' + - method: POST + endpoint: '/api/aslan/environment/environments/:name/services/?*/restartNew' + - method: POST + endpoint: '/api/aslan/environment/environments/:name/services/?*/scale' + - method: POST + endpoint: '/api/aslan/environment/environments/:name/services/?*/scaleNew' + - method: PUT + endpoint: '/api/aslan/environment/environments/:name/services/?*' + - method: POST + endpoint: '/api/aslan/environment/image/deployment/:name' + - method: POST + endpoint: '/api/aslan/environment/image/statefulset/:name' + - method: DELETE + endpoint: '/api/aslan/environment/kube/:name/pods/?*' + - method: PUT + endpoint: /api/aslan/environment/configmaps + - method: POST + endpoint: /api/aslan/environment/configmaps + - method: POST + endpoint: /api/aslan/workflow/servicetask + - action: delete_environment + alias: 删除 + description: '' + rules: + - method: DELETE + endpoint: '/api/aslan/environment/environments/:name' + - action: debug_pod + alias: 服务调试 + description: '' + rules: + - method: GET + endpoint: '/api/podexec/?*/?*/?*/?*/podExec/:name' + - method: POST + endpoint: '/api/aslan/environment/kube/:name/pods/?*/debugcontainer' + - action: ssh_pm + alias: 主机登录 + description: '' + rules: + - method: GET + endpoint: '/api/aslan/environment/environments/:name/services/?*/pmexec' + - resource: Scan + alias: "代码扫描" + description: "" + rules: + - action: get_scan + alias: 查看 + description: "" + rules: + - method: GET + endpoint: "api/aslan/testing/scanning" + - action: create_scan + alias: 新建 + description: "" + rules: + - method: POST + endpoint: "/api/aslan/testing/scanning" + - method: GET + endpoint: "/api/aslan/cluster/clusters" + - action: edit_scan + alias: 编辑 + description: "" + rules: + - method: PUT + endpoint: "/api/aslan/testing/scanning/?*" + - action: delete_scan + alias: "删除" + description: "" + rules: + - method: DELETE + endpoint: "/api/aslan/testing/scanning/?*" + - action: run_scan + alias: "执行" + description: '' + rules: + - method: POST + endpoint: "/api/aslan/testing/scanning/?*/task" diff --git a/pkg/microservice/policy/core/yamlconfig/role.go b/pkg/microservice/policy/core/yamlconfig/role.go new file mode 100644 index 0000000000000000000000000000000000000000..de3e927d4d762956b1a3f00742fc9afa16039a01 --- /dev/null +++ b/pkg/microservice/policy/core/yamlconfig/role.go @@ -0,0 +1,37 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package yamlconfig + +import _ "embed" + +//go:embed role.yaml +var roles []byte +var useConfigMapRoles bool + +var configMapRoles []byte + +func PresetRolesBytes() []byte { + if useConfigMapRoles { + return configMapRoles + } + return roles +} + +func RefreshRoles(b []byte) { + useConfigMapRoles = true + configMapRoles = b +} diff --git a/pkg/microservice/policy/core/yamlconfig/role.yaml b/pkg/microservice/policy/core/yamlconfig/role.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3dcbf17ba4c80810056aed7187cd2aba984ea2a4 --- /dev/null +++ b/pkg/microservice/policy/core/yamlconfig/role.yaml @@ -0,0 +1,111 @@ +description: "we have 5 preset roles: you can change the role rules to update permission" +preset_roles: + - name: admin + desc: 拥有系统中任何操作的权限 + rules: + - verbs: + - '*' + resources: + - '*' + - name: contributor + rules: + - verbs: + - get_workflow + - run_workflow + resources: + - Workflow + - verbs: + - get_environment + - config_environment + - manage_environment + - delete_environment + resources: + - Environment + - verbs: + - get_service + resources: + - Service + - verbs: + - get_build + resources: + - Build + - verbs: + - get_test + resources: + - Test + - name: project-admin + rules: + - verbs: + - '*' + resources: + - '*' + - name: read-only + rules: + - verbs: + - get_scan + resources: + - Scan + - verbs: + - get_workflow + resources: + - Workflow + - verbs: + - get_environment + resources: + - Environment + - verbs: + - get_environment + resources: + - ProductionEnvironment + - verbs: + - get_service + resources: + - Service + - verbs: + - get_build + resources: + - Build + - verbs: + - get_test + resources: + - Test + - verbs: + - get_delivery + resources: + - Delivery + - name: read-project-only + rules: + - verbs: + - get_workflow + resources: + - Workflow + match_attributes: + - key: placeholder + value: placeholder + - verbs: + - get_environment + resources: + - Environment + match_attributes: + - key: placeholder + value: placeholder + - verbs: + - get_service + resources: + - Service + - verbs: + - get_build + resources: + - Build + - verbs: + - get_test + resources: + - Test + - verbs: + - get_delivery + resources: + - Delivery + - verbs: + - get_scan + resources: + - Scan diff --git a/pkg/microservice/policy/core/yamlconfig/urls.go b/pkg/microservice/policy/core/yamlconfig/urls.go new file mode 100644 index 0000000000000000000000000000000000000000..99d22d0c68511cd9cd1ef796f26bea8c078f134c --- /dev/null +++ b/pkg/microservice/policy/core/yamlconfig/urls.go @@ -0,0 +1,75 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package yamlconfig + +import ( + _ "embed" + + "sigs.k8s.io/yaml" + + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types" +) + +//go:embed urls.yaml +var urls []byte +var defaultEmbedUrlConfig *ConfigYaml +var configMapUrlConfig *ConfigYaml +var useConfigMapUrls bool + +type ConfigYaml struct { + ExemptionUrls *ExemptionURLs `json:"exemption_urls"` + Description string `json:"description"` +} + +type ExemptionURLs struct { + Public []*types.PolicyRule `json:"public"` + SystemAdmin []*types.PolicyRule `json:"system_admin"` + ProjectAdmin []*types.PolicyRule `json:"project_admin"` +} + +func init() { + log.Init(&log.Config{ + Level: "debug", + NoCaller: true, + }) + if err := yaml.Unmarshal(urls, &defaultEmbedUrlConfig); err != nil { + log.DPanic(err) + } +} + +func GetExemptionsUrls() *ExemptionURLs { + if !useConfigMapUrls || configMapUrlConfig == nil { + return defaultEmbedUrlConfig.ExemptionUrls + } + return configMapUrlConfig.ExemptionUrls +} + +func GetDefaultEmbedUrlConfig() *ConfigYaml { + return defaultEmbedUrlConfig +} + +func RefreshConfigMapByte(b []byte) error { + if err := yaml.Unmarshal(b, &configMapUrlConfig); err != nil { + log.Errorf("refresh err:%s", err) + useConfigMapUrls = false + return err + } + log.Infof("start to use configMap url") + useConfigMapUrls = true + return nil +} diff --git a/pkg/microservice/policy/core/yamlconfig/urls.yaml b/pkg/microservice/policy/core/yamlconfig/urls.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3339b8482fc0a5641a1aa2c36fc765c4a129f603 --- /dev/null +++ b/pkg/microservice/policy/core/yamlconfig/urls.yaml @@ -0,0 +1,450 @@ +description: "public urls: url not need to authentication ; system_admin urls: urls that only system admin has permission ; project_admin: urls that project admin has permission" +exemption_urls: + public: + - endpoint: api/plutus/version/new/check + methods: + - GET + - endpoint: api/plutus/license/sync + methods: + - POST + - endpoint: api/plutus/customer/uploadBaseUri + methods: + - POST + - endpoint: api/plutus/customer/uploadCurrentVersion + methods: + - POST + - endpoint: api/plutus/version/latest + methods: + - GET + - endpoint: api/plutus/license/check + methods: + - POST + - endpoint: license + methods: + - GET + - endpoint: api/plutus/signature + methods: + - POST + - endpoint: api/plutus/** + methods: + - GET + - endpoint: plutus-vendor/** + methods: + - GET + - endpoint: plutus/deliveryBoard + methods: + - GET + - endpoint: api/aslan/environment/environments/?*/services/?*/devmode/patch + methods: + - POST + - endpoint: api/aslan/environment/environments/?*/services/?*/devmode/recover + methods: + - POST + - endpoint: api/v1/permission/project/?* + methods: + - GET + - endpoint: api/aslan/health + methods: + - GET + - endpoint: api/aslan/webhook + methods: + - POST + - endpoint: api/hub/connect + methods: + - GET + - endpoint: api/aslan/system/registry + methods: + - GET + - endpoint: api/aslan/testing/report + methods: + - GET + - endpoint: api/aslan/cluster/agent/?*/agent.yaml + methods: + - GET + - endpoint: api/podexec/health + methods: + - GET + - endpoint: api/v1/login + methods: + - GET + - POST + - endpoint: api/v1/signup + methods: + - GET + - POST + - endpoint: api/v1/retrieve + methods: + - GET + - POST + - endpoint: api/v1/login-enabled + methods: + - GET + - POST + - endpoint: login/password + methods: + - GET + - POST + - endpoint: api/v1/codehosts/?*/auth + methods: + - GET + - endpoint: api/v1/codehosts/callback + methods: + - GET + - endpoint: api/v1/callback + methods: + - GET + - POST + - PUT + - PATCH + - DELETE + - endpoint: dex/** + methods: + - GET + - POST + - PUT + - PATCH + - DELETE +# endpoint = '' is p0 level + - endpoint: '' + methods: + - GET + - endpoint: signin + methods: + - GET + - endpoint: setup + methods: + - GET + - endpoint: static/** + methods: + - GET + - endpoint: v1/** + methods: + - GET + - endpoint: mobile/** + methods: + - GET + - endpoint: workflows/** + methods: + - GET + - endpoint: favicon.ico + methods: + - GET + - endpoint: api/directory/codehosts/callback + methods: + - GET + - endpoint: api/v1/features/?* + methods: + - GET + - endpoint: api/aslan/system/jenkins/exist + methods: + - GET + - endpoint: api/callback + methods: + - POST + - endpoint: api/aslan/environment/environments/?*/check/workloads/k8services + methods: + - GET + - endpoint: api/aslan/environment/environments/?*/check/sharenv/enable/ready + methods: + - GET + - endpoint: api/aslan/environment/environments/?*/check/sharenv/disable/ready + methods: + - GET + - endpoint: api/aslan/service/services/?*/environments/deployable + methods: + - GET + system_admin: + - endpoint: api/v1/features/?* + methods: + - PUT + - endpoint: api/aslan/project/products + methods: + - POST + - endpoint: api/aslan/system/cleanCache/cron + methods: + - POST + - endpoint: api/aslan/system/sonar/?* + methods: + - POST + - PUT + - DELETE + - endpoint: api/v1/picket/projects + methods: + - POST + - endpoint: api/aslan/cluster/clusters + methods: + - POST + - endpoint: api/aslan/cluster/agent/?*/upgrade + methods: + - GET + - endpoint: api/aslan/cluster/clusters/?* + methods: + - PUT + - DELETE + - endpoint: api/aslan/cluster/clusters/?*/disconnect + methods: + - PUT + - endpoint: api/aslan/cluster/clusters/?*/reconnect + methods: + - PUT + - endpoint: api/collaboration/collaborations + methods: + - GET + - POST + - PUT + - DELETE + - endpoint: api/aslan/system/install + methods: + - POST + - PUT + - endpoint: api/aslan/system/install/delete + methods: + - PUT + - endpoint: api/aslan/system/proxyManage + methods: + - POST + - endpoint: api/aslan/system/proxyManage/?* + methods: + - PUT + - DELETE + - endpoint: api/aslan/system/registry/namespaces + methods: + - POST + - endpoint: api/aslan/system/registry/namespaces/?* + methods: + - PUT + - DELETE + - endpoint: api/aslan/system/s3storage + methods: + - POST + - endpoint: api/aslan/system/s3storage/?* + methods: + - PUT + - DELETE + - endpoint: api/aslan/system/githubApp + methods: + - POST + - endpoint: api/aslan/system/githubApp/?* + methods: + - DELETE + - endpoint: api/aslan/system/jenkins/integration + methods: + - POST + - endpoint: api/aslan/system/jenkins/integration/?* + methods: + - PUT + - DELETE + - endpoint: api/aslan/system/jenkins/integration/user/connection + methods: + - POST + - endpoint: api/aslan/system/basicImages + methods: + - POST + - endpoint: api/aslan/system/basicImages/?* + methods: + - PUT + - DELETE + - endpoint: api/aslan/system/helm + methods: + - POST + - endpoint: api/aslan/system/helm/?* + methods: + - PUT + - DELETE + - endpoint: api/aslan/system/privateKey + methods: + - POST + - endpoint: api/aslan/system/privateKey/?* + methods: + - PUT + - DELETE + - endpoint: api/aslan/system/announcement + methods: + - POST + - endpoint: api/aslan/system/announcement/update + methods: + - PUT + - endpoint: api/aslan/system/announcement/all + methods: + - GET + - endpoint: api/aslan/system/announcement/?* + methods: + - DELETE + - endpoint: api/aslan/system/operation + methods: + - GET + - POST + - endpoint: api/aslan/system/operation/?* + methods: + - PUT + - endpoint: api/aslan/system/proxy/config + methods: + - GET + - endpoint: api/v1/users/?* + methods: + - GET + - PUT + - PATCH + - endpoint: api/v1/picket/users/?* + methods: + - DELETE + - endpoint: api/v1/users + methods: + - POST + - endpoint: api/v1/public-roles + methods: + - POST + - endpoint: api/v1/public-roles/?* + methods: + - PUT + - DELETE + - endpoint: api/v1/system-roles + methods: + - GET + - POST + - endpoint: api/v1/system-roles/?* + methods: + - DELETE + - endpoint: api/v1/public-policies + methods: + - POST + - endpoint: api/v1/public-policies/?* + methods: + - PUT + - DELETE + - endpoint: api/v1/system-policies + methods: + - GET + - POST + - endpoint: api/v1/system-policies/?* + methods: + - DELETE + - endpoint: api/v1/system-rolebindings + methods: + - GET + - POST + - endpoint: api/v1/system-rolebindings/?* + methods: + - DELETE + - endpoint: api/v1/system-policybindings + methods: + - GET + - POST + - endpoint: api/v1/system-policybindings/?* + methods: + - DELETE + - endpoint: api/aslan/environment/envcfgs + methods: + - GET + - endpoint: api/aslan/environment/envcfgs/?*/sync + methods: + - PUT + project_admin: + - endpoint: api/aslan/project/products + methods: + - PUT + - endpoint: api/v1/picket/projects/?* + methods: + - PUT + - DELETE + - endpoint: api/aslan/project/products/?* + methods: + - DELETE + - endpoint: api/v1/users + methods: + - GET + - endpoint: api/v1/users/search + methods: + - POST + - endpoint: api/collaboration/collaborations + methods: + - GET + - POST + - PUT + - DELETE + - endpoint: api/v1/roles + methods: + - GET + - endpoint: api/v1/roles/?* + methods: + - GET + - endpoint: api/v1/roles + methods: + - POST + - endpoint: api/v1/roles/?* + methods: + - PUT + - DELETE + - endpoint: api/v1/roles/bulk-delete + methods: + - POST + - endpoint: api/v1/public-roles + methods: + - GET + - endpoint: api/v1/public-roles/?* + methods: + - GET + - endpoint: api/v1/policies + methods: + - GET + - endpoint: api/v1/policies/?* + methods: + - GET + - endpoint: api/v1/policies + methods: + - POST + - DELETE + - endpoint: api/v1/policies/?* + methods: + - PUT + - DELETE + - endpoint: api/v1/policies/bulk-delete + methods: + - POST + - endpoint: api/v1/public-policies + methods: + - GET + - endpoint: api/v1/public-policies/?* + methods: + - GET + - endpoint: api/v1/rolebindings + methods: + - GET + - POST + - endpoint: api/v1/picket/rolebindings + methods: + - GET + - endpoint: api/v1/rolebindings/?* + methods: + - DELETE + - endpoint: api/v1/rolebindings/bulk-delete + methods: + - POST + - endpoint: api/v1/policybindings + methods: + - GET + - POST + - endpoint: api/v1/picket/policybindings + methods: + - GET + - endpoint: api/v1/policybindings/?* + methods: + - DELETE + - endpoint: api/v1/policybindings/bulk-delete + methods: + - POST + - endpoint: api/aslan/project/pms/?* + methods: + - GET + - PUT + - DELETE + - endpoint: api/aslan/project/pms/labels + methods: + - GET + - endpoint: api/aslan/project/pms/batch + methods: + - POST + - endpoint: api/aslan/project/pms + methods: + - GET + - POST diff --git a/pkg/microservice/policy/server/rest/router.go b/pkg/microservice/policy/server/rest/router.go index 2115e654f22c2d7327b3e3e99f71697a7c53250c..d9fb530c969990ddb861607c44d41575fdd25106 100644 --- a/pkg/microservice/policy/server/rest/router.go +++ b/pkg/microservice/policy/server/rest/router.go @@ -23,10 +23,10 @@ import ( ) func (s *engine) injectRouterGroup(router *gin.RouterGroup) { - for name, r := range map[string]injector{ - "/api/v1": new(handler.Router), + for _, r := range []injector{ + new(handler.Router), } { - r.Inject(router.Group(name)) + r.Inject(router.Group("/api/v1")) } } diff --git a/pkg/microservice/policy/server/rest/server.go b/pkg/microservice/policy/server/rest/server.go index 345e1b43288ed41d070fad6b336df0eaa6b9f80b..2156f21b8ba802f57b62684b138f616172a3c3a5 100644 --- a/pkg/microservice/policy/server/rest/server.go +++ b/pkg/microservice/policy/server/rest/server.go @@ -55,7 +55,7 @@ func (s *engine) injectMiddlewares() { g.Use(ginmiddleware.Response()) g.Use(ginmiddleware.RequestID()) g.Use(ginmiddleware.RequestLog(log.NewFileLogger(config.RequestLogFile()))) - g.Use(RefreshOPABundle()) + //g.Use(RefreshOPABundle()) g.Use(gin.Recovery()) } diff --git a/pkg/microservice/policy/server/server.go b/pkg/microservice/policy/server/server.go index 9fb22f2362480fe1d9785f5615f6050a604c804d..e3a671f3d7026e7b78ccf292f9e67d4c2d9f4b5d 100644 --- a/pkg/microservice/policy/server/server.go +++ b/pkg/microservice/policy/server/server.go @@ -21,8 +21,25 @@ import ( "net/http" "time" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/tools/cache" + "sigs.k8s.io/yaml" + + "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/client" + + commonconfig "github.com/koderover/zadig/pkg/config" "github.com/koderover/zadig/pkg/microservice/policy/core" + "github.com/koderover/zadig/pkg/microservice/policy/core/service" + "github.com/koderover/zadig/pkg/microservice/policy/core/yamlconfig" "github.com/koderover/zadig/pkg/microservice/policy/server/rest" + "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" + toolClient "github.com/koderover/zadig/pkg/tool/kube/client" + "github.com/koderover/zadig/pkg/tool/kube/getter" + "github.com/koderover/zadig/pkg/tool/kube/updater" "github.com/koderover/zadig/pkg/tool/log" ) @@ -48,6 +65,21 @@ func Serve(ctx context.Context) error { log.Errorf("Failed to stop server, error: %s", err) } }() + go func() { + if err := toolClient.Start(ctx); err != nil { + panic(err) + } + }() + + if err := migratePolicyMeta(); err != nil { + log.Errorf("fail to migrate policyMeta, err:%s", err) + return err + } + + if err := migrateRole(); err != nil { + log.Errorf("fail to migrate role , err:%s", err) + return err + } if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { log.Errorf("Failed to start http server, error: %s", err) @@ -58,3 +90,190 @@ func Serve(ctx context.Context) error { return nil } + +type PresetRoleConfigYaml struct { + PresetRoles []*service.Role `json:"preset_roles"` + Description string `json:"description"` +} + +func migrateRole() error { + + bs := yamlconfig.PresetRolesBytes() + config := PresetRoleConfigYaml{} + + if err := yaml.Unmarshal(bs, &config); err != nil { + log.Errorf("yaml Unmarshal err:%s", err) + return err + } + + for _, role := range config.PresetRoles { + if role.Name == "admin" { + role.Namespace = "*" + } + for _, rule := range role.Rules { + rule.Kind = "resource" + } + role.Type = "system" + } + + for _, role := range config.PresetRoles { + if err := service.UpdateOrCreateRole(role.Namespace, role, nil); err != nil { + log.Errorf("UpdateOrCreateRole err:%s", err) + return err + } + } + return nil + +} + +// migratePolicyMeta migrate the policy meta db date +func migratePolicyMeta() error { + client, err := kubeclient.GetKubeClient(commonconfig.HubServerServiceAddress(), setting.LocalClusterID) + if err != nil { + log.DPanic(err) + } + clientset, err := kubeclient.GetKubeClientSet(commonconfig.HubServerServiceAddress(), setting.LocalClusterID) + if err != nil { + log.DPanic(err) + } + + namespace := commonconfig.Namespace() + policyMetasConfig := yamlconfig.DefaultPolicyMetasConfig() + policyMetaConfigBytes, err := yaml.Marshal(policyMetasConfig) + if err != nil { + return err + } + urls := yamlconfig.GetDefaultEmbedUrlConfig() + urlsBytes, err := yaml.Marshal(urls) + if err != nil { + return err + } + + metaConfigMap := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: setting.PolicyMetaConfigMapName, + Namespace: namespace, + }, + Data: map[string]string{ + "meta.yaml": string(policyMetaConfigBytes), + }, + } + + urlConfigMap := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: setting.PolicyURLConfigMapName, + Namespace: namespace, + }, + Data: map[string]string{ + "urls.yaml": string(urlsBytes), + }, + } + roleConfigMap := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: setting.PolicyRoleConfigMapName, + Namespace: namespace, + }, + Data: map[string]string{ + "roles.yaml": string(yamlconfig.PresetRolesBytes()), + }, + } + + if err := initConfigMap(metaConfigMap, client, clientset); err != nil { + log.Errorf("init config map meta err:%s", err) + } + if err := initConfigMap(urlConfigMap, client, clientset); err != nil { + log.Errorf("init config map url err:%s", err) + } + + if err := initConfigMap(roleConfigMap, client, clientset); err != nil { + log.Errorf("init config map roles err:%s", err) + } + + for _, v := range yamlconfig.DefaultPolicyMetas() { + if err := service.CreateOrUpdatePolicyRegistration(v, nil); err != nil { + log.DPanic(err) + } + } + _, err = NewClusterInformerFactory("", clientset) + return err +} + +func initConfigMap(cm *corev1.ConfigMap, client client.Client, clientset *kubernetes.Clientset) error { + _, found, err := getter.GetConfigMap(cm.Namespace, cm.Name, client) + if err != nil { + log.Errorf("get config map err:%s", err) + } + if !found { + err := updater.CreateConfigMap(cm, client) + if err != nil { + log.Infof("create config map err:%s", err) + return err + } + } else { + if err := updater.UpdateConfigMap(cm.Namespace, cm, clientset); err != nil { + log.Infof("update config map err:%s", err) + return err + } + } + return nil +} + +func NewClusterInformerFactory(clusterId string, cls *kubernetes.Clientset) (informers.SharedInformerFactory, error) { + if clusterId == "" { + clusterId = setting.LocalClusterID + } + informerFactory := informers.NewSharedInformerFactoryWithOptions(cls, time.Hour*24, informers.WithNamespace(commonconfig.Namespace())) + configMapsInformer := informerFactory.Core().V1().ConfigMaps().Informer() + configMapsInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + UpdateFunc: func(oldObj, newObj interface{}) { + configMap, ok := newObj.(*corev1.ConfigMap) + if !ok { + return + } + if configMap.Name == setting.PolicyMetaConfigMapName { + if b, ok := configMap.Data["meta.yaml"]; ok { + log.Infof("start to update the meta configMap") + for _, v := range yamlconfig.PolicyMetasFromBytes([]byte(b)) { + if err := service.CreateOrUpdatePolicyRegistration(v, nil); err != nil { + log.Errorf("fail to CreateOrUpdatePolicyRegistration,err:%s", err) + continue + } + } + } + } + if configMap.Name == setting.PolicyURLConfigMapName { + if b, ok := configMap.Data["urls.yaml"]; ok { + log.Infof("start to refresh url configMap data") + if err := yamlconfig.RefreshConfigMapByte([]byte(b)); err != nil { + log.Errorf("refresh urls err:%s", err) + } + } + } + if configMap.Name == setting.PolicyRoleConfigMapName { + if b, ok := configMap.Data["roles.yaml"]; ok { + log.Infof("start to refresh role configmap data") + yamlconfig.RefreshRoles([]byte(b)) + if err := migrateRole(); err != nil { + log.Errorf("refresh role err:%s", err) + } + } + } + + }, + }) + stop := make(chan struct{}) + informerFactory.Start(stop) + return informerFactory, nil +} diff --git a/pkg/microservice/predator/core/service/predator.go b/pkg/microservice/predator/core/service/predator.go index d1ad23fc827b13fd1b3f3514662ea602b9056f64..4325216047878d3bf9f2508ec24fcc2aaec1b7f3 100644 --- a/pkg/microservice/predator/core/service/predator.go +++ b/pkg/microservice/predator/core/service/predator.go @@ -18,11 +18,12 @@ package service // Context ... type Context struct { - JobType string `yaml:"job_type"` - DockerRegistry *DockerRegistry `yaml:"docker_registry"` - DockerBuildCtx *DockerBuildCtx `yaml:"build_ctx"` - OnSetup string `yaml:"setup,omitempty"` - ReleaseImages []RepoImage `yaml:"release_images"` + JobType string `yaml:"job_type"` + DockerRegistry *DockerRegistry `yaml:"docker_registry"` + DockerBuildCtx *DockerBuildCtx `yaml:"build_ctx"` + OnSetup string `yaml:"setup,omitempty"` + ReleaseImages []RepoImage `yaml:"release_images"` + DistributeList []*DistributeInfo `yaml:"distribute_info"` } type RepoImage struct { @@ -34,6 +35,24 @@ type RepoImage struct { Namespace string `yaml:"namespace"` } +// DistributeInfo will be convert into yaml, adding yaml +type DistributeInfo struct { + Image string `yaml:"image"` + ReleaseName string `yaml:"release_name"` + DistributeStartTime int64 `yaml:"distribute_start_time"` + DistributeEndTime int64 `yaml:"distribute_end_time"` + DistributeStatus string `yaml:"distribute_status"` + DeployEnabled bool `yaml:"deploy_enabled"` + DeployEnv string `yaml:"deploy_env"` + DeployNamespace string `yaml:"deploy_namespace"` + DeployStartTime int64 `yaml:"deploy_start_time"` + DeployEndTime int64 `yaml:"deploy_end_time"` + DeployStatus string `yaml:"deploy_status"` + RepoID string `yaml:"repo_id"` + RepoAK string `yaml:"repo_ak"` + RepoSK string `yaml:"repo_sk"` +} + // DockerBuildCtx ... // Docker build参数 // WorkDir: docker build执行路径 diff --git a/pkg/microservice/predator/core/service/service.go b/pkg/microservice/predator/core/service/service.go index a728fbbd549ae3ea12d0f0102145aa1a7495da19..35a3a4eb84534a6b037cbc07bd1d6a03556bfdc5 100644 --- a/pkg/microservice/predator/core/service/service.go +++ b/pkg/microservice/predator/core/service/service.go @@ -102,13 +102,15 @@ func (p *Predator) Exec() error { // AfterExec ... func (p *Predator) AfterExec() error { - for _, image := range p.Ctx.ReleaseImages { - err := writeDockerConfig(image.Host, image.Username, image.Password) + for _, distribute := range p.Ctx.DistributeList { + // get the docker host info from the image + splitString := strings.Split(distribute.Image, "/") + err := writeDockerConfig(splitString[0], distribute.RepoAK, distribute.RepoSK) if err != nil { return fmt.Errorf("failed to write docker config file %v", err) } - cmd := dockerPush(image.Name) + cmd := dockerPush(distribute.Image) cmd.Stderr = os.Stderr cmd.Stdout = os.Stdout cmd.Dir = p.Ctx.DockerBuildCtx.WorkDir @@ -161,8 +163,9 @@ func (p *Predator) dockerCommands() []*exec.Cmd { }) } else if p.Ctx.JobType == setting.ReleaseImageJob { cmds = append(cmds, dockerPull(p.Ctx.DockerBuildCtx.ImageName)) - for _, image := range p.Ctx.ReleaseImages { - cmds = append(cmds, dockerTag(p.Ctx.DockerBuildCtx.ImageName, image.Name)) + // no more ReleaseImages in release_image job type + for _, distribute := range p.Ctx.DistributeList { + cmds = append(cmds, dockerTag(p.Ctx.DockerBuildCtx.ImageName, distribute.Image)) } } diff --git a/pkg/microservice/reaper/core/service/archive/file_archive.go b/pkg/microservice/reaper/core/service/archive/file_archive.go index fc915ce0c3139f8695c719a075e1183748efadd1..6124542d5c704832d5d5d17b1800c69c7472fc11 100644 --- a/pkg/microservice/reaper/core/service/archive/file_archive.go +++ b/pkg/microservice/reaper/core/service/archive/file_archive.go @@ -41,16 +41,20 @@ type WorkspaceAchiever struct { StorageURI string PipelineName string ServiceName string + Envs []string + aesKey string } -func NewWorkspaceAchiever(storageURI, pipelineName, serviceName, wd string, caches []string, gitFolders []string) *WorkspaceAchiever { +func NewWorkspaceAchiever(storageURI, pipelineName, serviceName, wd, aesKey string, caches, gitFolders, envs []string) *WorkspaceAchiever { return &WorkspaceAchiever{ paths: caches, wd: wd, + aesKey: aesKey, gitFolders: gitFolders, StorageURI: storageURI, PipelineName: pipelineName, ServiceName: serviceName, + Envs: envs, } } @@ -117,8 +121,15 @@ func (c *WorkspaceAchiever) process(match string) bool { return false } -func (c *WorkspaceAchiever) processPaths(paths []string, verbose bool) { +func (c *WorkspaceAchiever) processPaths(paths []string, verbose bool) error { + newPath := make([]string, 0, len(paths)) for _, path := range paths { + path, err := c.renderPathVariable(path) + if err != nil { + log.Errorf("The variable of the custom cache directory is invalid, failed to render variables, error :%s", err) + return err + } + newPath = append(newPath, path) matches, err := filepath.Glob(filepath.Join(c.wd, path)) if err != nil { log.Warningf("%s: %v", path, err) @@ -147,12 +158,16 @@ func (c *WorkspaceAchiever) processPaths(paths []string, verbose bool) { } } } + c.paths = newPath + return nil } func (c *WorkspaceAchiever) enumerate() error { c.files = make(map[string]os.FileInfo) - c.processPaths(c.paths, true) + if err := c.processPaths(c.paths, true); err != nil { + return err + } for _, folder := range c.gitFolders { c.processPaths([]string{filepath.Join(folder, ".git")}, false) @@ -161,14 +176,14 @@ func (c *WorkspaceAchiever) enumerate() error { return nil } -func (c *WorkspaceAchiever) Achieve(target string) error { +func (c *WorkspaceAchiever) Achieve(target string) ([]string, error) { if err := c.enumerate(); err != nil { - return err + return nil, err } f, err := ioutil.TempFile("", "*cached_files.txt") if err != nil { - return err + return nil, err } defer func() { @@ -179,18 +194,18 @@ func (c *WorkspaceAchiever) Achieve(target string) error { for _, path := range c.sortedFiles() { _, err = f.WriteString(path + "\n") if err != nil { - return fmt.Errorf("failed to create cache file list: %s", err) + return nil, fmt.Errorf("failed to create cache file list: %s", err) } } if err := os.MkdirAll(filepath.Dir(target), os.ModePerm); err != nil { - return err + return nil, err } temp, err := ioutil.TempFile("", "*reaper.tar.gz") if err != nil { log.Errorf("failed to create temporary file: %v", err) - return err + return nil, err } _ = temp.Close() @@ -203,7 +218,7 @@ func (c *WorkspaceAchiever) Achieve(target string) error { if err := cmd.Run(); err != nil { log.Errorf("failed to compress %v", err) - return err + return nil, err } //if err := helper.Move(temp.Name(), target); err != nil { @@ -211,7 +226,7 @@ func (c *WorkspaceAchiever) Achieve(target string) error { // return err //} - if store, err := s3.NewS3StorageFromEncryptedURI(c.StorageURI); err == nil { + if store, err := s3.NewS3StorageFromEncryptedURI(c.StorageURI, c.aesKey); err == nil { forcedPathStyle := true if store.Provider == setting.ProviderSourceAli { forcedPathStyle = false @@ -219,15 +234,25 @@ func (c *WorkspaceAchiever) Achieve(target string) error { s3client, err := s3tool.NewClient(store.Endpoint, store.Ak, store.Sk, store.Insecure, forcedPathStyle) if err != nil { log.Errorf("Archive s3 create s3 client error: %+v", err) - return err + return nil, err } objectKey := store.GetObjectPath(fmt.Sprintf("%s/%s/%s/%s", c.PipelineName, c.ServiceName, "cache", meta.FileName)) if err = s3client.Upload(store.Bucket, temp.Name(), objectKey); err != nil { log.Errorf("Archive s3 upload err:%v", err) - return err + return nil, err } } - return nil + return c.paths, nil +} + +// The path contains shell variables, use linux's own ability to render +func (c *WorkspaceAchiever) renderPathVariable(path string) (string, error) { + envs := c.Envs + cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("%s %s", "echo", path)) + cmd.Stderr = os.Stderr + cmd.Env = envs + data, err := cmd.Output() + return strings.TrimSuffix(string(data), "\n"), err } diff --git a/pkg/microservice/reaper/core/service/meta/const.go b/pkg/microservice/reaper/core/service/meta/const.go index d7d8fb663c93d661a37c6bda8ff794bea79f8bb5..fe306fd640fb88597f85ad6d7b4de03c7024ad12 100644 --- a/pkg/microservice/reaper/core/service/meta/const.go +++ b/pkg/microservice/reaper/core/service/meta/const.go @@ -22,14 +22,18 @@ const ( // ProviderGitlab ... ProviderGitlab = "gitlab" - //ProviderIlyshin - ProviderIlyshin = "ilyshin" // ProviderGerrit ProviderGerrit = "gerrit" // ProviderCodehub ProviderCodehub = "codehub" + // ProviderGitee + ProviderGitee = "gitee" + + // ProviderOther + ProviderOther = "other" + // Oauth prefix OauthTokenPrefix = "oauth2" diff --git a/pkg/microservice/reaper/core/service/meta/types.go b/pkg/microservice/reaper/core/service/meta/types.go index 7dfd43373a4c73e8f39071c50e1b1d4c48facc51..e3612120d292599998f22d7948e9dd28ddb2f9a7 100644 --- a/pkg/microservice/reaper/core/service/meta/types.go +++ b/pkg/microservice/reaper/core/service/meta/types.go @@ -24,13 +24,17 @@ import ( "strings" "github.com/koderover/zadig/pkg/microservice/reaper/config" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/types" ) -// Context ... type Context struct { + // API token 服务访问使用的api token + APIToken string `yaml:"api_token"` // Workspace 容器工作目录 [必填] Workspace string `yaml:"workspace"` + // TODO: Deprecated. // CleanWorkspace 是否清理工作目录 [选填, 默认为 false] CleanWorkspace bool `yaml:"clean_workspace"` @@ -82,6 +86,7 @@ type Context struct { // Git Github/Gitlab 配置 Git *Git `yaml:"git"` + // TODO: Deprecated. // Caches Caches配置 Caches []string `yaml:"caches"` @@ -106,11 +111,39 @@ type Context struct { // IgnoreCache ignore docker build cache [runtime] IgnoreCache bool `yaml:"ignore_cache"` - StorageEndpoint string `yaml:"storage_endpoint"` - StorageAK string `yaml:"storage_ak"` - StorageSK string `yaml:"storage_sk"` - StorageBucket string `yaml:"storage_bucket"` - StorageProvider int `yaml:"storage_provider"` + StorageEndpoint string `yaml:"storage_endpoint"` + StorageAK string `yaml:"storage_ak"` + StorageSK string `yaml:"storage_sk"` + StorageBucket string `yaml:"storage_bucket"` + StorageProvider int `yaml:"storage_provider"` + ArtifactInfo *ArtifactInfo `yaml:"artifact_info"` + ArtifactPath string `yaml:"artifact_path"` + AesKey string `yaml:"aes_key"` + + // New since V1.10.0. + CacheEnable bool `yaml:"cache_enable"` + Cache types.Cache `yaml:"cache"` + CacheDirType types.CacheDirType `yaml:"cache_dir_type"` + CacheUserDir string `yaml:"cache_user_dir"` + + // Upload To S3 related context + UploadEnabled bool `yaml:"upload_enabled"` + UploadStorageInfo *types.ObjectStorageInfo `yaml:"upload_storage_info"` + UploadInfo []*types.ObjectStoragePathDetail `yaml:"upload_info"` + + // scanner used flag + ScannerFlag bool `yaml:"scanner_flag"` + ScannerType string `yaml:"scanner_type"` + SonarParameter string `yaml:"sonar_parameter"` + SonarServer string `yaml:"sonar_server"` + SonarLogin string `yaml:"sonar_login"` +} + +type ArtifactInfo struct { + URL string `yaml:"url"` + WorkflowName string `yaml:"workflow_name"` + TaskID int64 `yaml:"task_id"` + FileName string `yaml:"file_name"` } // DockerBuildCtx ... @@ -119,14 +152,20 @@ type Context struct { // DockerFile: dockerfile名称, 默认为Dockerfile // ImageBuild: build image镜像全称, e.g. xxx.com/spock-release-candidates/image:tag type DockerBuildCtx struct { - WorkDir string `yaml:"work_dir" bson:"work_dir" json:"work_dir"` - DockerFile string `yaml:"docker_file" bson:"docker_file" json:"docker_file"` - ImageName string `yaml:"image_name" bson:"image_name" json:"image_name"` - BuildArgs string `yaml:"build_args" bson:"build_args" json:"build_args"` - ImageReleaseTag string `yaml:"image_release_tag,omitempty" bson:"image_release_tag,omitempty" json:"image_release_tag"` + Source string `yaml:"source" bson:"source" json:"source"` + WorkDir string `yaml:"work_dir" bson:"work_dir" json:"work_dir"` + DockerFile string `yaml:"docker_file" bson:"docker_file" json:"docker_file"` + ImageName string `yaml:"image_name" bson:"image_name" json:"image_name"` + BuildArgs string `yaml:"build_args" bson:"build_args" json:"build_args"` + ImageReleaseTag string `yaml:"image_release_tag,omitempty" bson:"image_release_tag,omitempty" json:"image_release_tag"` + DockerTemplateContent string `yaml:"docker_template_content" bson:"docker_template_content" json:"docker_template_content"` } func (c *DockerBuildCtx) GetDockerFile() string { + // if the source of the dockerfile is from template, we write our own dockerfile + if c.Source == setting.DockerfileSourceTemplate { + return fmt.Sprintf("/%s", setting.ZadigDockerfilePath) + } if c.DockerFile == "" { return "Dockerfile" } @@ -144,6 +183,7 @@ type SSH struct { Name string `json:"name"` UserName string `json:"user_name"` IP string `json:"ip"` + Port int64 `json:"port"` IsProd bool `json:"is_prod"` Label string `json:"label"` PrivateKey string `json:"private_key"` @@ -222,20 +262,25 @@ type Install struct { // Repo ... type Repo struct { - Source string `yaml:"source"` - Address string `yaml:"address"` - Owner string `yaml:"owner"` - Name string `yaml:"name"` - RemoteName string `yaml:"remote_name"` - Branch string `yaml:"branch"` - PR int `yaml:"pr"` - Tag string `yaml:"tag"` - CheckoutPath string `yaml:"checkout_path"` - SubModules bool `yaml:"submodules"` - OauthToken string `yaml:"oauthToken"` - User string `yaml:"username"` - Password string `yaml:"password"` - CheckoutRef string `yaml:"checkout_ref"` + Source string `yaml:"source"` + Address string `yaml:"address"` + Owner string `yaml:"owner"` + Name string `yaml:"name"` + Namespace string `yaml:"namespace"` + RemoteName string `yaml:"remote_name"` + Branch string `yaml:"branch"` + PR int `yaml:"pr"` + Tag string `yaml:"tag"` + CheckoutPath string `yaml:"checkout_path"` + SubModules bool `yaml:"submodules"` + OauthToken string `yaml:"oauthToken"` + User string `yaml:"username"` + Password string `yaml:"password"` + CheckoutRef string `yaml:"checkout_ref"` + EnableProxy bool `yaml:"enable_proxy"` + AuthType types.AuthType `yaml:"auth_type,omitempty"` + SSHKey string `yaml:"ssh_key,omitempty"` + PrivateAccessToken string `yaml:"private_access_token,omitempty"` } // PRRef returns refs format @@ -370,6 +415,8 @@ func (g *Git) SSHCloneURL(source, owner, name string) string { func (g *Git) HTTPSCloneURL(source, token, owner, name string) string { if strings.ToLower(source) == ProviderGitlab { return fmt.Sprintf("https://%s/%s/%s.git", g.GetGitlabHost(), owner, name) + } else if strings.ToLower(source) == ProviderGitee { + return fmt.Sprintf("https://%s:%s@%s/%s/%s.git", OauthTokenPrefix, token, "gitee.com", owner, name) } //return fmt.Sprintf("https://x-access-token:%s@%s/%s/%s.git", g.GetInstallationToken(owner), g.GetGithubHost(), owner, name) return fmt.Sprintf("https://x-access-token:%s@%s/%s/%s.git", token, g.GetGithubHost(), owner, name) @@ -379,7 +426,7 @@ func (g *Git) HTTPSCloneURL(source, token, owner, name string) string { // e.g. //https://oauth2:ACCESS_TOKEN@somegitlab.com/owner/name.git func (g *Git) OAuthCloneURL(source, token, address, owner, name, scheme string) string { - if strings.ToLower(source) == ProviderGitlab || strings.ToLower(source) == ProviderIlyshin { + if strings.ToLower(source) == ProviderGitlab || strings.ToLower(source) == ProviderOther { // address 需要传过来 return fmt.Sprintf("%s://%s:%s@%s/%s/%s.git", scheme, OauthTokenPrefix, token, address, owner, name) } diff --git a/pkg/microservice/reaper/core/service/reaper/archive.go b/pkg/microservice/reaper/core/service/reaper/archive.go index b729ce665dd8d55920a05aaf12889db5b7e99158..c2c136bff4220fbb69c6d394602ea15cd3f091fc 100644 --- a/pkg/microservice/reaper/core/service/reaper/archive.go +++ b/pkg/microservice/reaper/core/service/reaper/archive.go @@ -34,7 +34,7 @@ func (r *Reaper) archiveS3Files() (err error) { if r.Ctx.FileArchiveCtx != nil && r.Ctx.StorageURI != "" { var store *s3.S3 - if store, err = s3.NewS3StorageFromEncryptedURI(r.Ctx.StorageURI); err != nil { + if store, err = s3.NewS3StorageFromEncryptedURI(r.Ctx.StorageURI, r.Ctx.AesKey); err != nil { log.Errorf("failed to create s3 storage %s", r.Ctx.StorageURI) return } @@ -77,7 +77,7 @@ func (r *Reaper) archiveTestFiles() error { return nil } - store, err := s3.NewS3StorageFromEncryptedURI(r.Ctx.StorageURI) + store, err := s3.NewS3StorageFromEncryptedURI(r.Ctx.StorageURI, r.Ctx.AesKey) if err != nil { log.Errorf("failed to create s3 storage %s, err: %s", r.Ctx.StorageURI, err) return err @@ -130,7 +130,7 @@ func (r *Reaper) archiveHTMLTestReportFile() error { return nil } - store, err := s3.NewS3StorageFromEncryptedURI(r.Ctx.StorageURI) + store, err := s3.NewS3StorageFromEncryptedURI(r.Ctx.StorageURI, r.Ctx.AesKey) if err != nil { log.Errorf("failed to create s3 storage %s, err: %s", r.Ctx.StorageURI, err) return err diff --git a/pkg/microservice/reaper/core/service/reaper/artifact.go b/pkg/microservice/reaper/core/service/reaper/artifact.go index ff93b4ff84faccbf28a460def596944ac00b5ea4..e046682ec4a40f600b52ae596fea77d8300dc66b 100644 --- a/pkg/microservice/reaper/core/service/reaper/artifact.go +++ b/pkg/microservice/reaper/core/service/reaper/artifact.go @@ -18,9 +18,8 @@ package reaper import ( "fmt" - "io/ioutil" "os" - "path" + "os/exec" "path/filepath" "strings" @@ -29,16 +28,17 @@ import ( "github.com/koderover/zadig/pkg/setting" "github.com/koderover/zadig/pkg/tool/log" s3tool "github.com/koderover/zadig/pkg/tool/s3" + "github.com/koderover/zadig/pkg/util/fs" ) -func artifactsUpload(ctx *meta.Context, activeWorkspace string) error { +func artifactsUpload(ctx *meta.Context, activeWorkspace string, artifactPaths []string, pluginType ...string) error { var ( err error store *s3.S3 ) if ctx.StorageURI != "" { - if store, err = s3.NewS3StorageFromEncryptedURI(ctx.StorageURI); err != nil { + if store, err = s3.NewS3StorageFromEncryptedURI(ctx.StorageURI, ctx.AesKey); err != nil { log.Errorf("artifactsUpload failed to create s3 storage err:%v", err) return err } @@ -49,50 +49,104 @@ func artifactsUpload(ctx *meta.Context, activeWorkspace string) error { } } - artifactPaths := ctx.GinkgoTest.ArtifactPaths - for _, artifactPath := range artifactPaths { - if len(artifactPath) == 0 { - continue + if len(pluginType) == 0 { + tarName := setting.ArtifactResultOut + cmdAndArtifactFullPaths := make([]string, 0) + cmdAndArtifactFullPaths = append(cmdAndArtifactFullPaths, "-czf") + cmdAndArtifactFullPaths = append(cmdAndArtifactFullPaths, tarName) + for _, artifactPath := range artifactPaths { + if len(artifactPath) == 0 { + continue + } + artifactPath = strings.TrimPrefix(artifactPath, "/") + artifactPath := filepath.Join(activeWorkspace, artifactPath) + isDir, err := fs.IsDir(artifactPath) + if err != nil || !isDir { + log.Errorf("artifactPath is not exist %s err: %s", artifactPath, err) + continue + } + cmdAndArtifactFullPaths = append(cmdAndArtifactFullPaths, artifactPath) } - artifactPath = strings.TrimPrefix(artifactPath, "/") - - artifactPath = filepath.Join(activeWorkspace, artifactPath) - - artifactFiles, err := ioutil.ReadDir(artifactPath) - if err != nil || len(artifactFiles) == 0 { - continue + if len(cmdAndArtifactFullPaths) < 3 { + return nil } - for _, artifactFile := range artifactFiles { - if artifactFile.IsDir() { - continue + temp, err := os.Create(tarName) + if err != nil { + log.Errorf("failed to create %s err: %s", tarName, err) + return err + } + _ = temp.Close() + cmd := exec.Command("tar", cmdAndArtifactFullPaths...) + cmd.Stderr = os.Stderr + if err = cmd.Run(); err != nil { + log.Errorf("failed to compress %s err:%s", tarName, err) + return err + } + if store != nil { + objectKey := store.GetObjectPath(fmt.Sprintf("%s/%s", activeWorkspace, tarName)) + if err = s3FileUpload(store, temp.Name(), objectKey); err != nil { + return err } - filePath := path.Join(artifactPath, artifactFile.Name()) + } + return nil + } - if _, err := os.Stat(filePath); os.IsNotExist(err) { - continue - } + artifactPath := filepath.Join(activeWorkspace, artifactPaths[0]) + isDir, err := fs.IsDir(artifactPath) + if err != nil { + return err + } + if isDir { + temp, err := os.CreateTemp("", "*artifact.tar.gz") + if err != nil { + log.Errorf("failed to create temp file %s", err) + return err + } - if store != nil { - forcedPathStyle := true - if store.Provider == setting.ProviderSourceAli { - forcedPathStyle = false - } - s3client, err := s3tool.NewClient(store.Endpoint, store.Ak, store.Sk, store.Insecure, forcedPathStyle) - if err != nil { - log.Errorf("failed to create s3 client, error is: %+v", err) - return err - } - objectKey := store.GetObjectPath(fmt.Sprintf("%s/%s", artifactPath, artifactFile.Name())) - err = s3client.Upload(store.Bucket, filePath, objectKey) - - if err != nil { - log.Errorf("artifactsUpload failed to upload package %s, err:%v", filePath, err) - return err - } + _ = temp.Close() + cmd := exec.Command("tar", "czf", temp.Name(), "-C", artifactPath, ".") + cmd.Dir = artifactPath + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + if err = cmd.Run(); err != nil { + log.Errorf("failed to compress artifact err:%s", err) + return err + } + if store != nil { + objectKey := store.GetObjectPath("artifact.tar.gz") + if err = s3FileUpload(store, temp.Name(), objectKey); err != nil { + return err + } + } + } else { + if store != nil { + _, fileName := filepath.Split(artifactPath) + objectKey := store.GetObjectPath(fileName) + if err = s3FileUpload(store, artifactPath, objectKey); err != nil { + return err } } } + + return nil +} + +func s3FileUpload(store *s3.S3, sourceFile, objectKey string) error { + forcedPathStyle := true + if store.Provider == setting.ProviderSourceAli { + forcedPathStyle = false + } + s3client, err := s3tool.NewClient(store.Endpoint, store.Ak, store.Sk, store.Insecure, forcedPathStyle) + if err != nil { + log.Errorf("failed to create s3 client, error is: %s", err) + return err + } + + if err = s3client.Upload(store.Bucket, sourceFile, objectKey); err != nil { + log.Errorf("Archive s3 upload err:%s", err) + return err + } return nil } diff --git a/pkg/microservice/reaper/core/service/reaper/cachemanager.go b/pkg/microservice/reaper/core/service/reaper/cachemanager.go index cac9050921eceeb3311527614a4ec6b4ada2c1f3..a4eba4d9ac9da5b4b6596c0671600450261f321f 100644 --- a/pkg/microservice/reaper/core/service/reaper/cachemanager.go +++ b/pkg/microservice/reaper/core/service/reaper/cachemanager.go @@ -80,13 +80,15 @@ type TarCacheManager struct { StorageURI string PipelineName string ServiceName string + aesKey string } -func NewTarCacheManager(storageURI, pipelineName, serviceName string) *TarCacheManager { +func NewTarCacheManager(storageURI, pipelineName, serviceName, aesKey string) *TarCacheManager { return &TarCacheManager{ StorageURI: storageURI, PipelineName: pipelineName, ServiceName: serviceName, + aesKey: aesKey, } } @@ -153,8 +155,7 @@ func (gcm *TarCacheManager) Unarchive(source, dest string) error { defer func() { _ = os.Remove(sourceFilename) }() - objectKey := store.GetObjectPath(files[0]) - err = s3client.Download(store.Bucket, objectKey, sourceFilename) + err = s3client.Download(store.Bucket, files[0], sourceFilename) if err != nil { return err } @@ -181,7 +182,7 @@ func (gcm *TarCacheManager) Unarchive(source, dest string) error { func (gcm *TarCacheManager) getS3Storage() (*s3.S3, error) { var err error var store *s3.S3 - if store, err = s3.NewS3StorageFromEncryptedURI(gcm.StorageURI); err != nil { + if store, err = s3.NewS3StorageFromEncryptedURI(gcm.StorageURI, gcm.aesKey); err != nil { log.Errorf("Archive failed to create s3 storage %s", gcm.StorageURI) return nil, err } diff --git a/pkg/microservice/reaper/core/service/reaper/docker.go b/pkg/microservice/reaper/core/service/reaper/docker.go index cfa97073bb3f84626eb82c267b543eb6861f54d0..ec5e62eebc5d3066d4a801043adee35560753aad 100644 --- a/pkg/microservice/reaper/core/service/reaper/docker.go +++ b/pkg/microservice/reaper/core/service/reaper/docker.go @@ -16,7 +16,9 @@ limitations under the License. package reaper -import "os/exec" +import ( + "os/exec" +) const dockerExe = "/usr/local/bin/docker" diff --git a/pkg/microservice/reaper/core/service/reaper/ginkgo.go b/pkg/microservice/reaper/core/service/reaper/ginkgo.go index 30c34682f4f6c83e88916e83a8500d5ae7350272..bff8b026c35a5345a6a73e1f23a33e119b98a5eb 100644 --- a/pkg/microservice/reaper/core/service/reaper/ginkgo.go +++ b/pkg/microservice/reaper/core/service/reaper/ginkgo.go @@ -49,7 +49,6 @@ func mergeGinkgoTestResults(testResultFile, testResultPath, testUploadPath strin ) if len(testResultPath) == 0 { - log.Warning("merge test result step skipped: no test result path found") return nil } diff --git a/pkg/microservice/reaper/core/service/reaper/git.go b/pkg/microservice/reaper/core/service/reaper/git.go index 4dbc6c0b8aba3f6b35723217d5ea8231d1b0cca2..f7ae043bd9127355e13bd37ae5a81fe91ffdc7e0 100644 --- a/pkg/microservice/reaper/core/service/reaper/git.go +++ b/pkg/microservice/reaper/core/service/reaper/git.go @@ -20,15 +20,20 @@ import ( "bufio" "encoding/base64" "fmt" + "io/ioutil" "net/url" "os" "path" "path/filepath" "strings" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/koderover/zadig/pkg/microservice/reaper/config" c "github.com/koderover/zadig/pkg/microservice/reaper/core/service/cmd" "github.com/koderover/zadig/pkg/microservice/reaper/core/service/meta" "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types" ) func (r *Reaper) RunGitGc(folder string) error { @@ -44,14 +49,17 @@ func (r *Reaper) RunGitGc(folder string) error { func (r *Reaper) runGitCmds() error { envs := r.getUserEnvs() - // 如果存在github代码库，则设置代理，同时保证非github库不走代理 if r.Ctx.Proxy.EnableRepoProxy && r.Ctx.Proxy.Type == "http" { noProxy := "" + proxyFlag := false for _, repo := range r.Ctx.Repos { - if repo.Source == meta.ProviderGithub { - envs = append(envs, fmt.Sprintf("http_proxy=%s", r.Ctx.Proxy.GetProxyURL())) - envs = append(envs, fmt.Sprintf("https_proxy=%s", r.Ctx.Proxy.GetProxyURL())) + if repo.EnableProxy { + if !proxyFlag { + envs = append(envs, fmt.Sprintf("http_proxy=%s", r.Ctx.Proxy.GetProxyURL())) + envs = append(envs, fmt.Sprintf("https_proxy=%s", r.Ctx.Proxy.GetProxyURL())) + proxyFlag = true + } } else { uri, err := url.Parse(repo.Address) if err == nil { @@ -79,7 +87,7 @@ func (r *Reaper) runGitCmds() error { //cmds = append(cmds, &c.Command{Cmd: c.SetConfig("http.postBuffer", "524288000"), DisableTrace: true}) cmds = append(cmds, &c.Command{Cmd: c.SetConfig("http.postBuffer", "2097152000"), DisableTrace: true}) var tokens []string - + var hostNames = sets.NewString() for _, repo := range r.Ctx.Repos { if repo == nil || len(repo.Name) == 0 { continue @@ -100,11 +108,18 @@ func (r *Reaper) runGitCmds() error { } } else if repo.Source == meta.ProviderCodehub { tokens = append(tokens, repo.Password) - } else if repo.Source == meta.ProviderIlyshin { - cmds = append(cmds, &c.Command{Cmd: c.SetConfig("http.sslVerify", "false"), DisableTrace: true}) + } else if repo.Source == meta.ProviderOther { + tokens = append(tokens, repo.PrivateAccessToken) + tokens = append(tokens, repo.SSHKey) } tokens = append(tokens, repo.OauthToken) - cmds = append(cmds, r.buildGitCommands(repo)...) + cmds = append(cmds, r.buildGitCommands(repo, hostNames)...) + } + // write ssh key + if len(hostNames.List()) > 0 { + if err := writeSSHConfigFile(hostNames, r.Ctx.Proxy); err != nil { + return err + } } for _, c := range cmds { @@ -146,7 +161,7 @@ func (r *Reaper) runGitCmds() error { return nil } -func (r *Reaper) buildGitCommands(repo *meta.Repo) []*c.Command { +func (r *Reaper) buildGitCommands(repo *meta.Repo, hostNames sets.String) []*c.Command { cmds := make([]*c.Command, 0) @@ -164,7 +179,14 @@ func (r *Reaper) buildGitCommands(repo *meta.Repo) []*c.Command { } // 预防非正常退出导致git被锁住 - _ = os.Remove(path.Join(workDir, "/.git/index.lock")) + indexLockPath := path.Join(workDir, "/.git/index.lock") + if err := os.RemoveAll(indexLockPath); err != nil { + log.Errorf("Failed to remove %s: %s", indexLockPath, err) + } + shallowLockPath := path.Join(workDir, "/.git/shallow.lock") + if err := os.RemoveAll(shallowLockPath); err != nil { + log.Errorf("Failed to remove %s: %s", shallowLockPath, err) + } if isDirEmpty(filepath.Join(workDir, ".git")) { cmds = append(cmds, &c.Command{Cmd: c.InitGit(workDir)}) @@ -172,10 +194,15 @@ func (r *Reaper) buildGitCommands(repo *meta.Repo) []*c.Command { cmds = append(cmds, &c.Command{Cmd: c.RemoteRemove(repo.RemoteName), DisableTrace: true, IgnoreError: true}) } - if repo.Source == meta.ProviderGitlab || repo.Source == meta.ProviderIlyshin { + // namespace represents the real owner + owner := repo.Namespace + if len(owner) == 0 { + owner = repo.Owner + } + if repo.Source == meta.ProviderGitlab { u, _ := url.Parse(repo.Address) cmds = append(cmds, &c.Command{ - Cmd: c.RemoteAdd(repo.RemoteName, r.Ctx.Git.OAuthCloneURL(repo.Source, repo.OauthToken, u.Host, repo.Owner, repo.Name, u.Scheme)), + Cmd: c.RemoteAdd(repo.RemoteName, r.Ctx.Git.OAuthCloneURL(repo.Source, repo.OauthToken, u.Host, owner, repo.Name, u.Scheme)), DisableTrace: true, }) } else if repo.Source == meta.ProviderGerrit { @@ -191,12 +218,43 @@ func (r *Reaper) buildGitCommands(repo *meta.Repo) []*c.Command { u, _ := url.Parse(repo.Address) user := url.QueryEscape(repo.User) cmds = append(cmds, &c.Command{ - Cmd: c.RemoteAdd(repo.RemoteName, fmt.Sprintf("%s://%s:%s@%s/%s/%s.git", u.Scheme, user, repo.Password, u.Host, repo.Owner, repo.Name)), + Cmd: c.RemoteAdd(repo.RemoteName, fmt.Sprintf("%s://%s:%s@%s/%s/%s.git", u.Scheme, user, repo.Password, u.Host, owner, repo.Name)), DisableTrace: true, }) + } else if repo.Source == meta.ProviderGitee { + cmds = append(cmds, &c.Command{Cmd: c.RemoteAdd(repo.RemoteName, r.Ctx.Git.HTTPSCloneURL(repo.Source, repo.OauthToken, repo.Owner, repo.Name)), DisableTrace: true}) + } else if repo.Source == meta.ProviderOther { + if repo.AuthType == types.SSHAuthType { + host := getHost(repo.Address) + if !hostNames.Has(host) { + if err := writeSSHFile(repo.SSHKey, host); err != nil { + log.Errorf("failed to write ssh file %s: %s", repo.SSHKey, err) + } + hostNames.Insert(host) + } + remoteName := fmt.Sprintf("%s:%s/%s.git", repo.Address, repo.Owner, repo.Name) + // Including the case of the port + if strings.Contains(repo.Address, ":") { + remoteName = fmt.Sprintf("%s/%s/%s.git", repo.Address, repo.Owner, repo.Name) + } + cmds = append(cmds, &c.Command{ + Cmd: c.RemoteAdd(repo.RemoteName, remoteName), + DisableTrace: true, + }) + } else if repo.AuthType == types.PrivateAccessTokenAuthType { + u, err := url.Parse(repo.Address) + if err != nil { + log.Errorf("failed to parse url,err:%s", err) + } else { + cmds = append(cmds, &c.Command{ + Cmd: c.RemoteAdd(repo.RemoteName, r.Ctx.Git.OAuthCloneURL(repo.Source, repo.PrivateAccessToken, u.Host, repo.Owner, repo.Name, u.Scheme)), + DisableTrace: true, + }) + } + } } else { // github - cmds = append(cmds, &c.Command{Cmd: c.RemoteAdd(repo.RemoteName, r.Ctx.Git.HTTPSCloneURL(repo.Source, repo.OauthToken, repo.Owner, repo.Name)), DisableTrace: true}) + cmds = append(cmds, &c.Command{Cmd: c.RemoteAdd(repo.RemoteName, r.Ctx.Git.HTTPSCloneURL(repo.Source, repo.OauthToken, owner, repo.Name)), DisableTrace: true}) } ref := repo.Ref() @@ -229,3 +287,42 @@ func (r *Reaper) buildGitCommands(repo *meta.Repo) []*c.Command { return cmds } + +func writeSSHFile(sshKey, hostName string) error { + if sshKey == "" { + return fmt.Errorf("ssh cannot be empty") + } + + if hostName == "" { + return fmt.Errorf("hostName cannot be empty") + } + + hostName = strings.Replace(hostName, ".", "", -1) + hostName = strings.Replace(hostName, ":", "", -1) + pathName := fmt.Sprintf("/.ssh/id_rsa.%s", hostName) + file := path.Join(config.Home(), pathName) + return ioutil.WriteFile(file, []byte(sshKey), 0400) +} + +func writeSSHConfigFile(hostNames sets.String, proxy *meta.Proxy) error { + out := "\nHOST *\nStrictHostKeyChecking=no\nUserKnownHostsFile=/dev/null\n" + for _, hostName := range hostNames.List() { + name := strings.Replace(hostName, ".", "", -1) + name = strings.Replace(name, ":", "", -1) + out += fmt.Sprintf("\nHost %s\nIdentityFile ~/.ssh/id_rsa.%s\n", hostName, name) + if proxy.EnableRepoProxy && proxy.Type == "socks5" { + out = out + fmt.Sprintf("ProxyCommand nc -x %s %%h %%p\n", proxy.GetProxyURL()) + } + } + file := path.Join(config.Home(), "/.ssh/config") + return ioutil.WriteFile(file, []byte(out), 0600) +} + +// git@github.com or git@github.com:2000 +// return github.com +func getHost(address string) string { + address = strings.TrimPrefix(address, "ssh://") + address = strings.TrimPrefix(address, "git@") + hostArr := strings.Split(address, ":") + return hostArr[0] +} diff --git a/pkg/microservice/reaper/core/service/reaper/jmeter.go b/pkg/microservice/reaper/core/service/reaper/jmeter.go index d7b4649b881e92226946d6a37a28079d0e37d844..54511c87c7e4eed484950f624ac7a179f014bf6e 100644 --- a/pkg/microservice/reaper/core/service/reaper/jmeter.go +++ b/pkg/microservice/reaper/core/service/reaper/jmeter.go @@ -31,7 +31,6 @@ import ( func JmeterTestResults(testResultFile, testResultPath, testUploadPath string) error { var err error if len(testResultPath) == 0 { - log.Warning("jmeter test result step skipped: no test result path found") return nil } diff --git a/pkg/microservice/reaper/core/service/reaper/reaper.go b/pkg/microservice/reaper/core/service/reaper/reaper.go index e720fb2ca186d2b38a847fac5e802a7ec6c75f0b..2973d99bea77c9acdf9097d0d4e8d0e1d5de15ab 100644 --- a/pkg/microservice/reaper/core/service/reaper/reaper.go +++ b/pkg/microservice/reaper/core/service/reaper/reaper.go @@ -19,6 +19,7 @@ package reaper import ( "bytes" "fmt" + "io" "io/ioutil" "os" "os/exec" @@ -27,29 +28,30 @@ import ( "strings" "time" + "github.com/koderover/zadig/pkg/tool/s3" "gopkg.in/yaml.v3" "github.com/koderover/zadig/pkg/microservice/reaper/config" - "github.com/koderover/zadig/pkg/microservice/reaper/core/service/archive" "github.com/koderover/zadig/pkg/microservice/reaper/core/service/meta" "github.com/koderover/zadig/pkg/setting" "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types" + "github.com/koderover/zadig/pkg/util/fs" ) const ( - // ReadmeScriptFile ... ReadmeScriptFile = "readme_script.sh" - // ReadmeFile ... - ReadmeFile = "/tmp/README" + ReadmeFile = "/tmp/README" ) -// Reaper ... type Reaper struct { Ctx *meta.Context StartTime time.Time ActiveWorkspace string - cm CacheManager - dogFeed bool + UserEnvs map[string]string + Type types.ReaperType + + cm CacheManager } func NewReaper() (*Reaper, error) { @@ -63,14 +65,39 @@ func NewReaper() (*Reaper, error) { return nil, fmt.Errorf("cannot unmarshal job data: %v", err) } - // 初始化容器Envs, 格式为: "key=value". - // ctx.Envs = os.Environ() - // 初始化容器Path ctx.Paths = config.Path() reaper := &Reaper{ Ctx: ctx, - cm: NewTarCacheManager(ctx.StorageURI, ctx.PipelineName, ctx.ServiceName), + cm: NewTarCacheManager(ctx.StorageURI, ctx.PipelineName, ctx.ServiceName, ctx.AesKey), + } + + if ctx.TestType != "" { + reaper.Type = types.TestReaperType + } else if ctx.ScannerFlag { + reaper.Type = types.ScanningReaperType + } else { + reaper.Type = types.BuildReaperType + } + + workspace := "/workspace" + if reaper.Ctx.ClassicBuild { + workspace = reaper.Ctx.Workspace + } + err = reaper.EnsureActiveWorkspace(workspace) + if err != nil { + return nil, fmt.Errorf("failed to ensure active workspace `%s`: %s", workspace, err) + } + + userEnvs := reaper.getUserEnvs() + reaper.UserEnvs = make(map[string]string, len(userEnvs)) + for _, env := range userEnvs { + items := strings.Split(env, "=") + if len(items) != 2 { + continue + } + + reaper.UserEnvs[items[0]] = items[1] } return reaper, nil @@ -80,35 +107,21 @@ func (r *Reaper) GetCacheFile() string { return filepath.Join(r.Ctx.Workspace, "reaper.tar.gz") } -func (r *Reaper) archiveCustomCaches(wd, dest string, caches []string) error { - fileAchiever := archive.NewWorkspaceAchiever(r.Ctx.StorageURI, r.Ctx.PipelineName, r.Ctx.ServiceName, wd, caches, []string{}) - - // list files matches caches - return fileAchiever.Achieve(dest) -} - func (r *Reaper) CompressCache(storageURI string) error { - err := r.EnsureActiveWorkspace(r.ActiveWorkspace) - if err != nil { - log.Errorf("EnsureActiveWorkspace err:%v", err) - return err + cacheDir := r.ActiveWorkspace + if r.Ctx.CacheDirType == types.UserDefinedCacheDir { + // Note: Product supports using environment variables, so we need to parsing the directory path here. + cacheDir = r.renderUserEnv(r.Ctx.CacheUserDir) } - if len(r.Ctx.Caches) > 0 { - log.Infof("custom caches will be cached: %v", r.Ctx.Caches) - if err := r.archiveCustomCaches(r.ActiveWorkspace, r.GetCacheFile(), r.Ctx.Caches); err != nil { - return err - } - log.Infof("succeed to cache %s", r.Ctx.Caches) - } else { - log.Infof("workspace will be cached in background") - if err := r.cm.Archive(r.ActiveWorkspace, r.GetCacheFile()); err != nil { - return err - } - log.Info("succeed to cache workspace") + + log.Infof("Data in `%s` will be cached.", cacheDir) + if err := r.cm.Archive(cacheDir, r.GetCacheFile()); err != nil { + return fmt.Errorf("failed to cache %s: %s", cacheDir, err) } + log.Infof("Succeed to cache %s.", cacheDir) // remove workspace - err = os.RemoveAll(r.ActiveWorkspace) + err := os.RemoveAll(r.ActiveWorkspace) if err != nil { log.Errorf("RemoveAll err:%v", err) return err @@ -117,16 +130,25 @@ func (r *Reaper) CompressCache(storageURI string) error { } func (r *Reaper) DecompressCache() error { - _ = r.EnsureActiveWorkspace(r.ActiveWorkspace) - if err := r.cm.Unarchive(r.GetCacheFile(), r.ActiveWorkspace); err != nil { - if strings.Contains(err.Error(), "decompression OK") { - // could met decompression OK, trailing garbage ignored - return nil - } - return err + cacheDir := r.ActiveWorkspace + if r.Ctx.CacheDirType == types.UserDefinedCacheDir { + // Note: Product supports using environment variables, so we need to parsing the directory path here. + cacheDir = r.renderUserEnv(r.Ctx.CacheUserDir) } - return nil + err := r.EnsureDir(cacheDir) + if err != nil { + return fmt.Errorf("failed to ensure cache dir `%s`: %s", cacheDir, err) + } + + log.Infof("Cache will be decompressed to %s.", cacheDir) + err = r.cm.Unarchive(r.GetCacheFile(), cacheDir) + if err != nil && strings.Contains(err.Error(), "decompression OK") { + // could met decompression OK, trailing garbage ignored + err = nil + } + + return err } func (r *Reaper) EnsureActiveWorkspace(workspace string) error { @@ -138,116 +160,99 @@ func (r *Reaper) EnsureActiveWorkspace(workspace string) error { r.ActiveWorkspace = tempWorkspace return os.Chdir(r.ActiveWorkspace) } + err := os.MkdirAll(workspace, os.ModePerm) if err != nil { return fmt.Errorf("failed to create workspace: %v", err) } r.ActiveWorkspace = workspace + return os.Chdir(r.ActiveWorkspace) } -// BeforeExec ... -func (r *Reaper) BeforeExec() error { - workspace := "/workspace" - - if r.Ctx.ClassicBuild { - workspace = r.Ctx.Workspace - } +func (r *Reaper) EnsureDir(dir string) error { + return os.MkdirAll(dir, os.ModePerm) +} +func (r *Reaper) BeforeExec() error { r.StartTime = time.Now() - if err := os.RemoveAll(workspace); err != nil { - log.Warning(err.Error()) - } - - if err := r.EnsureActiveWorkspace(workspace); err != nil { - return err - } - - log.Info("wait for docker daemon to start ...") - for i := 0; i < 15; i++ { - if err := dockerInfo().Run(); err == nil { - break + // the execution preparation are not required, so we skip it if the reaper type is scanning + if r.Type != types.ScanningReaperType { + log.Infof("Checking Docker Connectivity.") + startTimeCheckDocker := time.Now() + for i := 0; i < 15; i++ { + if err := dockerInfo().Run(); err == nil { + break + } + time.Sleep(time.Second * 1) + } + log.Infof("Check ended. Duration: %.2f seconds.", time.Since(startTimeCheckDocker).Seconds()) + + if r.Ctx.DockerRegistry != nil { + if r.Ctx.DockerRegistry.UserName != "" { + log.Infof("Logining Docker Registry: %s.", r.Ctx.DockerRegistry.Host) + startTimeDockerLogin := time.Now() + cmd := dockerLogin(r.Ctx.DockerRegistry.UserName, r.Ctx.DockerRegistry.Password, r.Ctx.DockerRegistry.Host) + var out bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &out + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to login docker registry: %s %s", err, out.String()) + } + + log.Infof("Login ended. Duration: %.2f seconds.", time.Since(startTimeDockerLogin).Seconds()) + } } - time.Sleep(time.Second * 1) - } - // 检查是否需要登录docker registry - if r.Ctx.DockerRegistry != nil { - if r.Ctx.DockerRegistry.UserName != "" { - log.Infof("login docker registry %s", r.Ctx.DockerRegistry.Host) - cmd := dockerLogin(r.Ctx.DockerRegistry.UserName, r.Ctx.DockerRegistry.Password, r.Ctx.DockerRegistry.Host) - var out bytes.Buffer - cmd.Stdout = &out - cmd.Stderr = &out - if err := cmd.Run(); err != nil { - log.Errorf("docker login failed with error: %s\n%s", err, out.String()) - return fmt.Errorf("docker login failed with error: %s", err) + if r.Ctx.CacheEnable && r.Ctx.Cache.MediumType == types.ObjectMedium { + log.Info("Pulling Cache.") + startTimePullCache := time.Now() + if err := r.DecompressCache(); err != nil { + // If the workflow runs for the first time, there may be no cache. + log.Infof("Failed to pull cache: %s. Duration: %.2f seconds.", err, time.Since(startTimePullCache).Seconds()) + } else { + log.Infof("Succeed to pull cache. Duration: %.2f seconds.", time.Since(startTimePullCache).Seconds()) } } - } - // CleanWorkspace=True 意思是不使用缓存，ResetCache=True 意思是当次工作流不使用缓存 - // 如果 CleanWorkspace=True，永远不使用缓存 - // 如果 CleanWorkspace=False，本次工作流 ResetCache=False，使用缓存；本次工作流 ResetCache=True，不使用缓存 - // TODO: CleanWorkspace 和 ResetCache 严重词不达意，需要改成更合理的值 - if !r.Ctx.CleanWorkspace && !r.Ctx.ResetCache { - // 恢复缓存 - //if _, err := os.Stat(r.GetCacheFile()); err == nil { - // 解压缓存 - log.Info("extracting workspace ...") - if err := r.DecompressCache(); err != nil { - log.Infof("no previous cache is found: %v", err) - //if err = os.Remove(r.GetCacheFile()); err != nil { - // log.Warningf("failed to remove cache file %s: %v", r.GetCacheFile(), err) - //} - } else { - log.Info("succeed to extract workspace") + if err := os.MkdirAll(path.Join(os.Getenv("HOME"), "/.ssh"), os.ModePerm); err != nil { + return fmt.Errorf("create ssh folder error: %v", err) } - //} - } - - // 创建SSH目录 - if err := os.MkdirAll(path.Join(os.Getenv("HOME"), "/.ssh"), os.ModePerm); err != nil { - return fmt.Errorf("create ssh folder error: %v", err) - } - // 创建发布目录 - if r.Ctx.Archive != nil && len(r.Ctx.Archive.Dir) > 0 { - if err := os.MkdirAll(r.Ctx.Archive.Dir, os.ModePerm); err != nil { - return fmt.Errorf("create DistDir error: %v", err) + if r.Ctx.Archive != nil && len(r.Ctx.Archive.Dir) > 0 { + if err := os.MkdirAll(r.Ctx.Archive.Dir, os.ModePerm); err != nil { + return fmt.Errorf("create DistDir error: %v", err) + } } - } - // 检查是否需要配置Gitub/Gitlab - if r.Ctx.Git != nil { - if err := r.Ctx.Git.WriteGithubSSHFile(); err != nil { - return fmt.Errorf("write github ssh file error: %v", err) - } + if r.Ctx.Git != nil { + if err := r.Ctx.Git.WriteGithubSSHFile(); err != nil { + return fmt.Errorf("write github ssh file error: %v", err) + } - if err := r.Ctx.Git.WriteGitlabSSHFile(); err != nil { - return fmt.Errorf("write gitlab ssh file error: %v", err) - } + if err := r.Ctx.Git.WriteGitlabSSHFile(); err != nil { + return fmt.Errorf("write gitlab ssh file error: %v", err) + } - if err := r.Ctx.Git.WriteKnownHostFile(); err != nil { - return fmt.Errorf("write known_host file error: %v", err) - } + if err := r.Ctx.Git.WriteKnownHostFile(); err != nil { + return fmt.Errorf("write known_host file error: %v", err) + } - if err := r.Ctx.Git.WriteSSHConfigFile(r.Ctx.Proxy); err != nil { - return fmt.Errorf("write ssh config error: %v", err) + if err := r.Ctx.Git.WriteSSHConfigFile(r.Ctx.Proxy); err != nil { + return fmt.Errorf("write ssh config error: %v", err) + } } - } - // 清理测试目录 - if r.Ctx.GinkgoTest != nil && len(r.Ctx.GinkgoTest.ResultPath) > 0 { - r.Ctx.GinkgoTest.ResultPath = filepath.Join(r.ActiveWorkspace, r.Ctx.GinkgoTest.ResultPath) - log.Infof("clean test result path %s", r.Ctx.GinkgoTest.ResultPath) - if err := os.RemoveAll(r.Ctx.GinkgoTest.ResultPath); err != nil { - log.Warning(err.Error()) - } - // 创建测试目录 - if err := os.MkdirAll(r.Ctx.GinkgoTest.ResultPath, os.ModePerm); err != nil { - return fmt.Errorf("create test result path error: %v", err) + if r.Ctx.GinkgoTest != nil && len(r.Ctx.GinkgoTest.ResultPath) > 0 { + r.Ctx.GinkgoTest.ResultPath = filepath.Join(r.ActiveWorkspace, r.Ctx.GinkgoTest.ResultPath) + if err := os.RemoveAll(r.Ctx.GinkgoTest.ResultPath); err != nil { + log.Warning(err.Error()) + } + + if err := os.MkdirAll(r.Ctx.GinkgoTest.ResultPath, os.ModePerm); err != nil { + return fmt.Errorf("create test result path error: %v", err) + } } } @@ -255,21 +260,23 @@ func (r *Reaper) BeforeExec() error { } func dockerBuildCmd(dockerfile, fullImage, ctx, buildArgs string, ignoreCache bool) *exec.Cmd { - args := []string{"build", "--rm=true"} + args := []string{"-c"} + dockerCommand := "docker build --rm=true" if ignoreCache { - args = append(args, "--no-cache") + dockerCommand += " --no-cache" } if buildArgs != "" { for _, val := range strings.Fields(buildArgs) { if val != "" { - args = append(args, val) + dockerCommand = dockerCommand + " " + val } } } - args = append(args, []string{"-t", fullImage, "-f", dockerfile, ctx}...) - return exec.Command(dockerExe, args...) + dockerCommand = dockerCommand + " -t " + fullImage + " -f " + dockerfile + " " + ctx + args = append(args, dockerCommand) + return exec.Command("sh", args...) } func (r *Reaper) setProxy(ctx *meta.DockerBuildCtx, cfg *meta.Proxy) { @@ -300,145 +307,201 @@ func (r *Reaper) dockerCommands() []*exec.Cmd { } func (r *Reaper) runDockerBuild() error { - if r.Ctx.DockerBuildCtx != nil { - if r.Ctx.Proxy != nil { - r.setProxy(r.Ctx.DockerBuildCtx, r.Ctx.Proxy) - } - - envs := r.getUserEnvs() - for _, c := range r.dockerCommands() { - c.Stdout = os.Stdout - c.Stderr = os.Stderr - c.Dir = r.ActiveWorkspace - c.Env = envs - if err := c.Run(); err != nil { - return err - } + if r.Ctx.DockerBuildCtx == nil { + return nil + } + + log.Info("Preparing Dockerfile.") + startTimePrepareDockerfile := time.Now() + err := r.prepareDockerfile() + if err != nil { + return fmt.Errorf("failed to prepare dockerfile: %s", err) + } + log.Infof("Preparation ended. Duration: %.2f seconds.", time.Since(startTimePrepareDockerfile).Seconds()) + + if r.Ctx.Proxy != nil { + r.setProxy(r.Ctx.DockerBuildCtx, r.Ctx.Proxy) + } + + log.Info("Runing Docker Build.") + startTimeDockerBuild := time.Now() + envs := r.getUserEnvs() + for _, c := range r.dockerCommands() { + c.Stdout = os.Stdout + c.Stderr = os.Stderr + c.Dir = r.ActiveWorkspace + c.Env = envs + if err := c.Run(); err != nil { + return fmt.Errorf("failed to run docker build: %s", err) } } + log.Infof("Docker build ended. Duration: %.2f seconds.", time.Since(startTimeDockerBuild).Seconds()) return nil } -// Exec ... -func (r *Reaper) Exec() error { +func (r *Reaper) prepareDockerfile() error { + if r.Ctx.DockerBuildCtx.Source == setting.DockerfileSourceTemplate { + reader := strings.NewReader(r.Ctx.DockerBuildCtx.DockerTemplateContent) + readCloser := io.NopCloser(reader) + path := fmt.Sprintf("/%s", setting.ZadigDockerfilePath) + err := fs.SaveFile(readCloser, path) + if err != nil { + return err + } + } + return nil +} - // 运行安装脚本 +func (r *Reaper) Exec() error { + log.Info("Installing Dependency Packages.") + startTimeInstallDeps := time.Now() if err := r.runIntallationScripts(); err != nil { - return err + return fmt.Errorf("failed to install dependency packages: %s", err) } + log.Infof("Install ended. Duration: %.2f seconds.", time.Since(startTimeInstallDeps).Seconds()) - // 运行Git命令 + log.Info("Cloning Repository.") + startTimeCloneRepo := time.Now() if err := r.runGitCmds(); err != nil { - return err + return fmt.Errorf("failed to clone repository: %s", err) } + log.Infof("Clone ended. Duration: %.2f seconds.", time.Since(startTimeCloneRepo).Seconds()) - // 生成Git commits信息 if err := r.createReadme(ReadmeFile); err != nil { - log.Warningf("create readme file error: %v", err) + log.Warningf("Failed to create README file: %s", err) } - // 运行用户脚本 - if err := r.runScripts(); err != nil { - return err + // for build/test/scanner of type other than sonar, we run the script + if !r.Ctx.ScannerFlag || r.Ctx.ScannerType != types.ScanningTypeSonar { + log.Info("Executing User Build Script.") + startTimeRunBuildScript := time.Now() + if err := r.runScripts(); err != nil { + return fmt.Errorf("failed to execute user build script: %s", err) + } + log.Infof("Execution ended. Duration: %.2f seconds.", time.Since(startTimeRunBuildScript).Seconds()) + } else { + // for sonar type we write the sonar parameter into config file and go with sonar-scanner command + log.Info("Executing SonarQube Scanning process.") + startTimeRunSonar := time.Now() + if err := r.runSonarScanner(); err != nil { + return fmt.Errorf("failed to execute sonar scanning process, the error is: %s", err) + } + log.Infof("Sonar scan ended. Duration %.2f seconds.", time.Since(startTimeRunSonar).Seconds()) } return r.runDockerBuild() } -// AfterExec ... -func (r *Reaper) AfterExec(upStreamErr error) error { - var err error - if r.Ctx.GinkgoTest != nil && r.Ctx.GinkgoTest.ResultPath != "" { +func (r *Reaper) AfterExec() error { + if r.Ctx.GinkgoTest != nil { resultPath := r.Ctx.GinkgoTest.ResultPath - if !strings.HasPrefix(resultPath, "/") { + if resultPath != "" && !strings.HasPrefix(resultPath, "/") { resultPath = filepath.Join(r.ActiveWorkspace, resultPath) } + if r.Ctx.TestType == "" { r.Ctx.TestType = setting.FunctionTest } - if r.Ctx.TestType == setting.FunctionTest { - log.Info("merging test result") - // 解析功能测试的测试结果目录的文件，对数据进行统计，将最终的统计结果写入到一个本地文件中 - if err = mergeGinkgoTestResults( - r.Ctx.Archive.File, - resultPath, - r.Ctx.Archive.Dir, - r.StartTime, - ); err != nil { - log.Errorf("function err %v", err) - return err + + switch r.Ctx.TestType { + case setting.FunctionTest: + err := mergeGinkgoTestResults(r.Ctx.Archive.File, resultPath, r.Ctx.Archive.Dir, r.StartTime) + if err != nil { + return fmt.Errorf("failed to merge test result: %s", err) } - } else if r.Ctx.TestType == setting.PerformanceTest { - log.Info("performance test result") - // 解析性能测试的测试结果目录的文件，对数据进行统计，将最终的统计结果写入到一个本地文件中 - if err = JmeterTestResults( - r.Ctx.Archive.File, - resultPath, - r.Ctx.Archive.Dir, - ); err != nil { - log.Errorf("performance err %v", err) - return err + case setting.PerformanceTest: + err := JmeterTestResults(r.Ctx.Archive.File, resultPath, r.Ctx.Archive.Dir) + if err != nil { + return fmt.Errorf("failed to archive performance test result: %s", err) } } - // 将测试文件导出地址的文件上传到S3 + if len(r.Ctx.GinkgoTest.ArtifactPaths) > 0 { - if err = artifactsUpload(r.Ctx, r.ActiveWorkspace); err != nil { - log.Errorf("artifactsUpload err %v", err) - return err + if err := artifactsUpload(r.Ctx, r.ActiveWorkspace, r.Ctx.GinkgoTest.ArtifactPaths); err != nil { + return fmt.Errorf("failed to upload artifacts: %s", err) } } - // 将上面生成的统计结果文件上传到S3 - if err = r.archiveTestFiles(); err != nil { - log.Errorf("archiveFiles err %v", err) - return err - } - // 将HTML测试报告上传到S3 - if err = r.archiveHTMLTestReportFile(); err != nil { - log.Errorf("archiveFiles err %v", err) - return err + + if err := r.archiveTestFiles(); err != nil { + return fmt.Errorf("failed to archive test files: %s", err) } + if err := r.archiveHTMLTestReportFile(); err != nil { + return fmt.Errorf("failed to archive HTML test report: %s", err) + } } - // should archive file first, since compress cache will clean the workspace - if upStreamErr == nil { - if err = r.archiveS3Files(); err != nil { - log.Errorf("archiveFiles err %v", err) - return err + if r.Ctx.ArtifactInfo == nil { + if err := r.archiveS3Files(); err != nil { + return fmt.Errorf("failed to archive S3 files: %s", err) } - // 运行构建后置脚本 - if err = r.RunPostScripts(); err != nil { - log.Errorf("RunPostScripts err %v", err) - return err + + if err := r.RunPostScripts(); err != nil { + return fmt.Errorf("failed to run postscripts: %s", err) + } + } else { + if err := r.downloadArtifactFile(); err != nil { + return fmt.Errorf("failed to download artifact files: %s", err) } + } - // 运行物理机部署脚本 - if err = r.RunPMDeployScripts(); err != nil { - log.Errorf("RunPMDeployScripts err %v", err) - return err + if r.Ctx.UploadEnabled { + forcedPathStyle := true + if r.Ctx.UploadStorageInfo.Provider == setting.ProviderSourceAli { + forcedPathStyle = false + } + client, err := s3.NewClient(r.Ctx.UploadStorageInfo.Endpoint, r.Ctx.UploadStorageInfo.AK, r.Ctx.UploadStorageInfo.SK, r.Ctx.UploadStorageInfo.Insecure, forcedPathStyle) + if err != nil { + return fmt.Errorf("failed to create s3 client to upload file, err: %s", err) } + for _, upload := range r.Ctx.UploadInfo { + info, err := os.Stat(upload.AbsFilePath) + if err != nil { + return fmt.Errorf("failed to upload file path [%s] to destination [%s], the error is: %s", upload.AbsFilePath, upload.DestinationPath, err) + } + // if the given path is a directory + if info.IsDir() { + err := client.UploadDir(r.Ctx.UploadStorageInfo.Bucket, upload.AbsFilePath, upload.DestinationPath) + if err != nil { + log.Errorf("Failed to upload dir [%s] to path [%s] on s3, the error is: %s", upload.AbsFilePath, upload.DestinationPath, err) + return err + } + } else { + key := filepath.Join(upload.DestinationPath, info.Name()) + err := client.Upload(r.Ctx.UploadStorageInfo.Bucket, upload.AbsFilePath, key) + if err != nil { + log.Errorf("Failed to upload [%s] to key [%s] on s3, the error is: %s", upload.AbsFilePath, key, err) + return err + } + } + } + } - // create dog food file to tell wd that task is finished - dogFoodErr := ioutil.WriteFile(setting.DogFood, []byte(time.Now().Format(time.RFC3339)), 0644) - if dogFoodErr != nil { - log.Infof("failed to create dog food %v", dogFoodErr) - } else { - // end here - r.dogFeed = true - log.Infof("build end. duration: %.2f seconds", time.Since(r.StartTime).Seconds()) + if r.Ctx.ArtifactPath != "" { + if err := artifactsUpload(r.Ctx, r.ActiveWorkspace, []string{r.Ctx.ArtifactPath}, "buildv3"); err != nil { + return fmt.Errorf("failed to upload artifacts: %s", err) } } - if upStreamErr == nil { - _ = r.CompressCache(r.Ctx.StorageURI) + if err := r.RunPMDeployScripts(); err != nil { + return fmt.Errorf("failed to run deploy scripts on physical machine: %s", err) } - return err -} + // Upload workspace cache if the user turns on caching and uses object storage. + // Note: Whether the cache is uploaded successfully or not cannot hinder the progress of the overall process, + // so only exceptions are printed here and the process is not interrupted. + if r.Ctx.CacheEnable && r.Ctx.Cache.MediumType == types.ObjectMedium { + log.Info("Uploading Build Cache.") + startTimeUploadBuildCache := time.Now() + if err := r.CompressCache(r.Ctx.StorageURI); err != nil { + log.Warnf("Failed to upload build cache: %s. Duration: %.2f seconds.", err, time.Since(startTimeUploadBuildCache).Seconds()) + } else { + log.Infof("Upload ended. Duration: %.2f seconds.", time.Since(startTimeUploadBuildCache).Seconds()) + } + } -func (r *Reaper) DogFeed() bool { - return r.dogFeed + return nil } func (r *Reaper) maskSecret(secrets []string, message string) string { @@ -497,3 +560,11 @@ func (r *Reaper) getUserEnvs() []string { return envs } + +func (r *Reaper) renderUserEnv(raw string) string { + mapper := func(env string) string { + return r.UserEnvs[env] + } + + return os.Expand(raw, mapper) +} diff --git a/pkg/microservice/reaper/core/service/reaper/script.go b/pkg/microservice/reaper/core/service/reaper/script.go index bbe55ba357b0b3f857b9b3ebacbe69f495ec4494..45f8a1a405190703a393a445c2bc7ad4fbf47e39 100644 --- a/pkg/microservice/reaper/core/service/reaper/script.go +++ b/pkg/microservice/reaper/core/service/reaper/script.go @@ -20,12 +20,15 @@ import ( "bufio" "encoding/base64" "fmt" + "io" + "io/fs" "io/ioutil" "os" "os/exec" "path" "path/filepath" "strings" + "sync" "github.com/koderover/zadig/pkg/microservice/reaper/config" "github.com/koderover/zadig/pkg/microservice/reaper/internal/s3" @@ -148,7 +151,6 @@ func (r *Reaper) runIntallationScripts() error { } func (r *Reaper) createReadme(file string) error { - if r.Ctx.Archive == nil || len(r.Ctx.Repos) == 0 { return nil } @@ -164,7 +166,6 @@ func (r *Reaper) createReadme(file string) error { scripts = append(scripts, fmt.Sprintf("echo GIT-COMMIT: >> %s", file)) for _, repo := range r.Ctx.Repos { - if repo == nil || len(repo.Name) == 0 { continue } @@ -206,10 +207,6 @@ func (r *Reaper) runScripts() error { // avoid non-blocking IO for stdout to workaround "stdout: write error" for _, script := range r.Ctx.Scripts { scripts = append(scripts, script) - if strings.Contains(script, "yarn ") || strings.Contains(script, "npm ") || strings.Contains(script, "bower ") { - scripts = append(scripts, "echo 'turn off O_NONBLOCK after using node'") - scripts = append(scripts, "python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);'") - } } userScriptFile := "user_script.sh" @@ -225,37 +222,82 @@ func (r *Reaper) runScripts() error { //如果文件不存在就创建文件，避免后面使用变量出错 util.WriteFile(fileName, []byte{}, 0700) - cmdOutReader, err := cmd.StdoutPipe() + needPersistentLog := len(r.Ctx.PostScripts) > 0 + + var wg sync.WaitGroup + + cmdStdoutReader, err := cmd.StdoutPipe() if err != nil { return err } - outScanner := bufio.NewScanner(cmdOutReader) + wg.Add(1) go func() { - for outScanner.Scan() { - fmt.Printf("%s\n", r.maskSecretEnvs(outScanner.Text())) - if len(r.Ctx.PostScripts) > 0 { - util.WriteFile(fileName, []byte(outScanner.Text()+"\n"), 0700) - } - } + defer wg.Done() + + r.handleCmdOutput(cmdStdoutReader, needPersistentLog, fileName) }() - cmdErrReader, err := cmd.StderrPipe() + cmdStdErrReader, err := cmd.StderrPipe() if err != nil { return err } - errScanner := bufio.NewScanner(cmdErrReader) + wg.Add(1) go func() { - for errScanner.Scan() { - fmt.Printf("%s\n", r.maskSecretEnvs(errScanner.Text())) - if len(r.Ctx.PostScripts) > 0 { - util.WriteFile(fileName, []byte(errScanner.Text()+"\n"), 0700) - } + defer wg.Done() + + r.handleCmdOutput(cmdStdErrReader, needPersistentLog, fileName) + }() + + if err := cmd.Start(); err != nil { + return err + } + + wg.Wait() + + return cmd.Wait() +} + +func (r *Reaper) runSonarScanner() error { + // first we write the sonar scanner config to the config file + for _, repo := range r.Ctx.Repos { + log.Infof("Writing sonar-project.properties for repo: %s", repo.Name) + repoConfigPath := filepath.Join("/workspace", repo.Name, "sonar-project.properties") + configContent := fmt.Sprintf("sonar.login=%s\nsonar.host.url=%s\n%s", r.Ctx.SonarLogin, r.Ctx.SonarServer, r.Ctx.SonarParameter) + err := os.WriteFile(repoConfigPath, []byte(configContent), fs.ModeAppend) + if err != nil { + log.Errorf("failed to write sonar-project.properties for repo: %s, the error is: %s", repo.Name, err) + return err } + } + // then we simply run the sonar-scanner command to commence the scan + cmd := exec.Command("sonar-scanner") + // since currently only one codehost is supported, we will just use the first repository + cmd.Dir = filepath.Join("/workspace", r.Ctx.Repos[0].Name) + fileName := filepath.Join(os.TempDir(), "sonar.log") + util.WriteFile(fileName, []byte{}, 0700) + var wg sync.WaitGroup + + cmdStdoutReader, err := cmd.StdoutPipe() + if err != nil { + return err + } + + wg.Add(1) + go func() { + defer wg.Done() + + r.handleCmdOutput(cmdStdoutReader, false, fileName) }() - return cmd.Run() + if err := cmd.Start(); err != nil { + return err + } + + wg.Wait() + + return cmd.Wait() } func (r *Reaper) prepareScriptsEnv() []string { @@ -342,6 +384,7 @@ func (r *Reaper) RunPMDeployScripts() error { cmd.Env = append(cmd.Env, fmt.Sprintf("%s_PK=%s", ssh.Name, filepath.Join(os.TempDir(), ssh.Name+"_PK"))) cmd.Env = append(cmd.Env, fmt.Sprintf("%s_IP=%s", ssh.Name, ssh.IP)) + cmd.Env = append(cmd.Env, fmt.Sprintf("%s_PORT=%d", ssh.Name, ssh.Port)) cmd.Env = append(cmd.Env, fmt.Sprintf("%s_USERNAME=%s", ssh.Name, ssh.UserName)) r.Ctx.SecretEnvs = append(r.Ctx.SecretEnvs, fmt.Sprintf("%s_PK=%s", ssh.Name, filepath.Join(os.TempDir(), ssh.Name+"_PK"))) @@ -373,3 +416,64 @@ func (r *Reaper) RunPMDeployScripts() error { return cmd.Run() } + +func (r *Reaper) downloadArtifactFile() error { + var err error + var store *s3.S3 + if store, err = s3.NewS3StorageFromEncryptedURI(r.Ctx.ArtifactInfo.URL, r.Ctx.AesKey); err != nil { + log.Errorf("Archive failed to create s3 storage %s", r.Ctx.ArtifactInfo.URL) + return err + } + if store.Subfolder != "" { + store.Subfolder = fmt.Sprintf("%s/%s/%d/%s", store.Subfolder, r.Ctx.ArtifactInfo.WorkflowName, r.Ctx.ArtifactInfo.TaskID, "file") + } else { + store.Subfolder = fmt.Sprintf("%s/%d/%s", r.Ctx.ArtifactInfo.WorkflowName, r.Ctx.ArtifactInfo.TaskID, "file") + } + + forcedPathStyle := true + if store.Provider == setting.ProviderSourceAli { + forcedPathStyle = false + } + s3client, err := s3tool.NewClient(store.Endpoint, store.Ak, store.Sk, store.Insecure, forcedPathStyle) + if err != nil { + log.Errorf("s3 create client err:%s", err) + return err + } + files, err := s3client.ListFiles(store.Bucket, store.GetObjectPath(r.Ctx.ArtifactInfo.FileName), false) + if err != nil { + log.Errorf("s3 list file err:%s", err) + return err + } + if len(files) > 0 { + if err = s3client.Download(store.Bucket, files[0], fmt.Sprintf("%s/%s", r.ActiveWorkspace, r.Ctx.ArtifactInfo.FileName)); err != nil { + log.Errorf("s3 download file err:%s", err) + return err + } + } + return nil +} + +func (r *Reaper) handleCmdOutput(pipe io.ReadCloser, needPersistentLog bool, logFile string) { + reader := bufio.NewReader(pipe) + + for { + lineBytes, err := reader.ReadBytes('\n') + if err != nil { + if err == io.EOF { + break + } + + log.Errorf("Failed to read log when processing cmd output: %s", err) + break + } + + fmt.Printf("%s", r.maskSecretEnvs(string(lineBytes))) + + if needPersistentLog { + err := util.WriteFile(logFile, lineBytes, 0700) + if err != nil { + log.Warnf("Failed to write file when processing cmd output: %s", err) + } + } + } +} diff --git a/pkg/microservice/reaper/executor/executor.go b/pkg/microservice/reaper/executor/executor.go index 15af03878a1daa44e048f58f37585aa78e2e78d7..d5f56ce02fd64fbc450ece40c355ee982fd1dcd9 100644 --- a/pkg/microservice/reaper/executor/executor.go +++ b/pkg/microservice/reaper/executor/executor.go @@ -17,14 +17,15 @@ limitations under the License. package executor import ( + "fmt" + "io/ioutil" "time" - "github.com/hashicorp/go-multierror" - commonconfig "github.com/koderover/zadig/pkg/config" "github.com/koderover/zadig/pkg/microservice/reaper/core/service/reaper" "github.com/koderover/zadig/pkg/setting" "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types" ) func Execute() error { @@ -36,30 +37,54 @@ func Execute() error { }) start := time.Now() - log.Info("build start") - - r, err := reaper.NewReaper() - if err != nil { - log.Fatal(err) - } + var err error + var reaperType types.ReaperType defer func() { - // when dog is feed, the following log has been printed - if !r.DogFeed() { - log.Infof("build end. duration: %.2f seconds", time.Since(start).Seconds()) + // Create dog food file to tell wd that task has finished. + resultMsg := types.JobSuccess + if err != nil { + resultMsg = types.JobFail + log.Errorf("Failed to run: %s.", err) } + log.Infof("Job Status: %s", resultMsg) + + dogFoodErr := ioutil.WriteFile(setting.DogFood, []byte(resultMsg), 0644) + if dogFoodErr != nil { + log.Errorf("Failed to create dog food: %s.", dogFoodErr) + } + + log.Infof("====================== %s End. Duration: %.2f seconds ======================", reaperType, time.Since(start).Seconds()) + + // Note: Mark the task has been completed through the dogfood file, indirectly notify wd to do follow-up + // operations, and wait for a fixed time. + // Since `wd` will automatically delete the job after detecting the dogfile, this time has little + // effect on the overall construction time. + time.Sleep(30 * time.Second) }() - var errs *multierror.Error + var r *reaper.Reaper + r, err = reaper.NewReaper() + if err != nil { + return fmt.Errorf("failed to new reaper: %s", err) + } + + reaperType = r.Type + log.Infof("====================== %s Start ======================", reaperType) + if err = r.BeforeExec(); err != nil { - log.Fatal(err) + return fmt.Errorf("failed to prepare before building: %s", err) } - if err = r.Exec(); err != nil { - errs = multierror.Append(errs, err) + + if r.Ctx.ArtifactInfo == nil { + if err = r.Exec(); err != nil { + return fmt.Errorf("failed to build: %s", err) + } } - if err = r.AfterExec(err); err != nil { - errs = multierror.Append(errs, err) + + if err = r.AfterExec(); err != nil { + return fmt.Errorf("failed to work after building: %s", err) } - return errs.ErrorOrNil() + return nil } diff --git a/pkg/microservice/reaper/internal/s3/s3.go b/pkg/microservice/reaper/internal/s3/s3.go index 3c15cc4b871e9ae053f8f5df28b72328b9dcf603..ed87e9c7b2bf31908dde1aa3c3332e05cec3ab7d 100644 --- a/pkg/microservice/reaper/internal/s3/s3.go +++ b/pkg/microservice/reaper/internal/s3/s3.go @@ -75,8 +75,8 @@ func NewS3StorageFromURL(uri string) (*S3, error) { return ret, nil } -func NewS3StorageFromEncryptedURI(encryptedURI string) (*S3, error) { - uri, err := crypto.AesDecrypt(encryptedURI) +func NewS3StorageFromEncryptedURI(encryptedURI, aesKey string) (*S3, error) { + uri, err := crypto.AesDecrypt(encryptedURI, aesKey) if err != nil { return nil, err } diff --git a/pkg/microservice/systemconfig/config/config.go b/pkg/microservice/systemconfig/config/config.go new file mode 100644 index 0000000000000000000000000000000000000000..bc3f164bdbb80fd88ab1b3f52d61261bcf59a714 --- /dev/null +++ b/pkg/microservice/systemconfig/config/config.go @@ -0,0 +1,39 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "github.com/spf13/viper" + + configbase "github.com/koderover/zadig/pkg/config" +) + +func MysqlDexDB() string { + return viper.GetString(ENVMysqlDexDB) +} + +func Features() string { + return viper.GetString(FeatureFlag) +} + +func MongoURI() string { + return configbase.MongoURI() +} + +func MongoDatabase() string { + return configbase.MongoDatabase() +} diff --git a/pkg/microservice/warpdrive/core/service/taskcontroller/const.go b/pkg/microservice/systemconfig/config/consts.go similarity index 85% rename from pkg/microservice/warpdrive/core/service/taskcontroller/const.go rename to pkg/microservice/systemconfig/config/consts.go index e7de59fde1a92ec51f8f9fb971c8f08f0cfb112d..476a94cc1ef95e9dacec31d5ad2e5d7c852c46c9 100644 --- a/pkg/microservice/warpdrive/core/service/taskcontroller/const.go +++ b/pkg/microservice/systemconfig/config/consts.go @@ -14,9 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. */ -package taskcontroller +package config const ( - // MaxWorkerInParallel is max worker concurrency - maxWorkerInParallel = 5 + ENVMysqlDexDB = "MYSQL_DEX_DB" + FeatureFlag = "feature-gates" ) diff --git a/pkg/microservice/systemconfig/core/codehost/handler/codehost.go b/pkg/microservice/systemconfig/core/codehost/handler/codehost.go new file mode 100644 index 0000000000000000000000000000000000000000..20cf7643ab17e1599adae580f05a947168c9895e --- /dev/null +++ b/pkg/microservice/systemconfig/core/codehost/handler/codehost.go @@ -0,0 +1,143 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "fmt" + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/systemconfig/core/codehost/repository/models" + "github.com/koderover/zadig/pkg/microservice/systemconfig/core/codehost/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +func CreateCodeHost(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + rep := new(models.CodeHost) + if err := c.ShouldBindJSON(rep); err != nil { + ctx.Err = err + return + } + ctx.Resp, ctx.Err = service.CreateCodeHost(rep, ctx.Logger) +} + +func ListCodeHost(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + encryptedKey := c.Query("encryptedKey") + if len(encryptedKey) == 0 { + ctx.Err = e.ErrInvalidParam + return + } + ctx.Resp, ctx.Err = service.List(encryptedKey, c.Query("address"), c.Query("owner"), c.Query("source"), ctx.Logger) +} + +func ListCodeHostInternal(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + ctx.Resp, ctx.Err = service.ListInternal(c.Query("address"), c.Query("owner"), c.Query("source"), ctx.Logger) +} + +func DeleteCodeHost(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + idParam := c.Param("id") + id, err := strconv.Atoi(idParam) + if err != nil { + ctx.Err = err + return + } + ctx.Err = service.DeleteCodeHost(id, ctx.Logger) +} + +func GetCodeHost(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + idParam := c.Param("id") + id, err := strconv.Atoi(idParam) + if err != nil { + ctx.Err = err + return + } + + ignoreDelete := false + if len(c.Query("ignoreDelete")) > 0 { + ignoreDelete, err = strconv.ParseBool(c.Query("ignoreDelete")) + if err != nil { + ctx.Err = fmt.Errorf("failed to parse param ignoreDelete, err: %s", err) + return + } + } + + ctx.Resp, ctx.Err = service.GetCodeHost(id, ignoreDelete, ctx.Logger) +} + +func AuthCodeHost(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + id := c.Param("id") + idInt, err := strconv.Atoi(id) + if err != nil { + ctx.Err = err + return + } + url, err := service.AuthCodeHost(c.Query("redirect_url"), idInt, ctx.Logger) + if err != nil { + ctx.Err = err + ctx.Logger.Errorf("auth err,id:%d,err: %s", idInt, err) + return + } + c.Redirect(http.StatusFound, url) +} + +func Callback(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + state := c.Query("state") + redirectURL, err := service.HandleCallback(state, c.Request, ctx.Logger) + if err != nil { + ctx.Logger.Errorf("Callback err:%s", err) + ctx.Err = err + return + } + c.Redirect(http.StatusFound, redirectURL) +} + +func UpdateCodeHost(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + idParam := c.Param("id") + id, err := strconv.Atoi(idParam) + if err != nil { + ctx.Err = err + return + } + req := &models.CodeHost{} + if err := c.ShouldBindJSON(req); err != nil { + ctx.Err = err + return + } + req.ID = id + ctx.Resp, ctx.Err = service.UpdateCodeHost(req, ctx.Logger) +} diff --git a/pkg/microservice/systemconfig/core/codehost/handler/router.go b/pkg/microservice/systemconfig/core/codehost/handler/router.go new file mode 100644 index 0000000000000000000000000000000000000000..128169abfae206e190e440c239f6dec81665881a --- /dev/null +++ b/pkg/microservice/systemconfig/core/codehost/handler/router.go @@ -0,0 +1,37 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" +) + +type Router struct{} + +func (*Router) Inject(router *gin.RouterGroup) { + codehost := router.Group("codehosts") + { + codehost.GET("/callback", Callback) + codehost.GET("", ListCodeHost) + codehost.GET("/internal", ListCodeHostInternal) + codehost.DELETE("/:id", DeleteCodeHost) + codehost.POST("", CreateCodeHost) + codehost.PATCH("/:id", UpdateCodeHost) + codehost.GET("/:id", GetCodeHost) + codehost.GET("/:id/auth", AuthCodeHost) + } +} diff --git a/pkg/microservice/systemconfig/core/codehost/internal/oauth/oauth.go b/pkg/microservice/systemconfig/core/codehost/internal/oauth/oauth.go new file mode 100644 index 0000000000000000000000000000000000000000..f7b17fa4a1e6eedf608c3fe2325f5d5b93870e1d --- /dev/null +++ b/pkg/microservice/systemconfig/core/codehost/internal/oauth/oauth.go @@ -0,0 +1,90 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oauth + +import ( + "context" + "fmt" + "net/http" + "net/url" + + "golang.org/x/oauth2" + + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/systemconfig/core/codehost/repository/models" + "github.com/koderover/zadig/pkg/tool/log" +) + +type OAuth struct { + oauth2Config *oauth2.Config +} + +type OAuth2Error struct { + Err string + Description string +} + +func (e *OAuth2Error) Error() string { + if e.Description == "" { + return e.Err + } + return fmt.Sprintf("%s: %s", e.Err, e.Description) +} + +func New(callbackURL, clientID, clientSecret string, scopes []string, endpoint oauth2.Endpoint) *OAuth { + return &OAuth{ + oauth2Config: &oauth2.Config{ + ClientID: clientID, + ClientSecret: clientSecret, + Endpoint: endpoint, + RedirectURL: callbackURL, + Scopes: scopes, + }, + } +} + +func (o *OAuth) LoginURL(state string) string { + return o.oauth2Config.AuthCodeURL(state) +} + +func (o *OAuth) HandleCallback(r *http.Request, c *models.CodeHost) (*oauth2.Token, error) { + q := r.URL.Query() + if errType := q.Get("error"); errType != "" { + return nil, &OAuth2Error{errType, q.Get("error_description")} + } + + httpClient := http.DefaultClient + // if set http proxy + proxies, err := commonrepo.NewProxyColl().List(&commonrepo.ProxyArgs{}) + if err == nil && len(proxies) != 0 && proxies[0].EnableRepoProxy && c.EnableProxy { + log.Info("use proxy") + port := proxies[0].Port + ip := proxies[0].Address + proxyRawUrl := fmt.Sprintf("http://%s:%d", ip, port) + proxyUrl, err2 := url.Parse(proxyRawUrl) + if err2 == nil { + httpClient.Transport = &http.Transport{ + Proxy: http.ProxyURL(proxyUrl), + } + } + + } + + ctx := context.Background() + ctx = context.WithValue(ctx, oauth2.HTTPClient, httpClient) + return o.oauth2Config.Exchange(ctx, q.Get("code")) +} diff --git a/pkg/microservice/systemconfig/core/codehost/repository/models/codehost.go b/pkg/microservice/systemconfig/core/codehost/repository/models/codehost.go new file mode 100644 index 0000000000000000000000000000000000000000..80a8f23e339d64e41c995ec14daac11c084bdb80 --- /dev/null +++ b/pkg/microservice/systemconfig/core/codehost/repository/models/codehost.go @@ -0,0 +1,48 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/koderover/zadig/pkg/types" +) + +type CodeHost struct { + ID int `bson:"id" json:"id"` + Type string `bson:"type" json:"type"` + Address string `bson:"address" json:"address"` + IsReady string `bson:"is_ready" json:"is_ready"` + AccessToken string `bson:"access_token" json:"access_token"` + RefreshToken string `bson:"refresh_token" json:"refresh_token"` + Namespace string `bson:"namespace" json:"namespace"` + ApplicationId string `bson:"application_id" json:"application_id"` + Region string `bson:"region,omitempty" json:"region,omitempty"` + Username string `bson:"username,omitempty" json:"username,omitempty"` + Password string `bson:"password,omitempty" json:"password,omitempty"` + ClientSecret string `bson:"client_secret" json:"client_secret"` + Alias string `bson:"alias,omitempty" json:"alias,omitempty"` + AuthType types.AuthType `bson:"auth_type,omitempty" json:"auth_type,omitempty"` + SSHKey string `bson:"ssh_key,omitempty" json:"ssh_key,omitempty"` + PrivateAccessToken string `bson:"private_access_token,omitempty" json:"private_access_token,omitempty"` + CreatedAt int64 `bson:"created_at" json:"created_at"` + UpdatedAt int64 `bson:"updated_at" json:"updated_at"` + DeletedAt int64 `bson:"deleted_at" json:"deleted_at"` + EnableProxy bool `bson:"enable_proxy" json:"enable_proxy"` +} + +func (CodeHost) TableName() string { + return "code_host" +} diff --git a/pkg/microservice/systemconfig/core/codehost/repository/mongodb/codehost.go b/pkg/microservice/systemconfig/core/codehost/repository/mongodb/codehost.go new file mode 100644 index 0000000000000000000000000000000000000000..71759aa5b2305c12699921aa308401a3570b74bb --- /dev/null +++ b/pkg/microservice/systemconfig/core/codehost/repository/mongodb/codehost.go @@ -0,0 +1,200 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + + "github.com/koderover/zadig/pkg/microservice/systemconfig/config" + "github.com/koderover/zadig/pkg/microservice/systemconfig/core/codehost/repository/models" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/log" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type CodehostColl struct { + *mongo.Collection + + coll string +} + +type ListArgs struct { + Owner string + Address string + Source string +} + +func NewCodehostColl() *CodehostColl { + name := models.CodeHost{}.TableName() + coll := &CodehostColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} + + return coll +} + +func (c *CodehostColl) GetCollectionName() string { + return c.coll +} +func (c *CodehostColl) EnsureIndex(ctx context.Context) error { + return nil +} + +func (c *CodehostColl) AddCodeHost(iCodeHost *models.CodeHost) (*models.CodeHost, error) { + + _, err := c.Collection.InsertOne(context.TODO(), iCodeHost) + if err != nil { + log.Error("repository AddCodeHost err : %v", err) + return nil, err + } + return iCodeHost, nil +} + +func (c *CodehostColl) DeleteCodeHost() error { + query := bson.M{"deleted_at": 0} + change := bson.M{"$set": bson.M{ + "deleted_at": time.Now().Unix(), + }} + + _, err := c.Collection.UpdateOne(context.TODO(), query, change) + if err != nil { + log.Error("repository DeleteCodeHostByID err : %v", err) + return err + } + return nil +} + +func (c *CodehostColl) GetCodeHostByAlias(alias string) (*models.CodeHost, error) { + codehost := new(models.CodeHost) + query := bson.M{"alias": alias, "deleted_at": 0} + if err := c.Collection.FindOne(context.TODO(), query).Decode(codehost); err != nil { + return nil, err + } + return codehost, nil +} + +func (c *CodehostColl) GetCodeHostByID(ID int, ignoreDelete bool) (*models.CodeHost, error) { + codehost := new(models.CodeHost) + query := bson.M{"id": ID} + if !ignoreDelete { + query["deleted_at"] = 0 + } + if err := c.Collection.FindOne(context.TODO(), query).Decode(codehost); err != nil { + return nil, err + } + return codehost, nil +} + +func (c *CodehostColl) List(args *ListArgs) ([]*models.CodeHost, error) { + codeHosts := make([]*models.CodeHost, 0) + query := bson.M{"deleted_at": 0} + + if args == nil { + args = &ListArgs{} + } + if args.Address != "" { + query["address"] = args.Address + } + if args.Owner != "" { + query["namespace"] = args.Owner + } + if args.Source != "" { + query["type"] = args.Source + } + + cursor, err := c.Collection.Find(context.TODO(), query) + if err != nil { + return nil, err + } + err = cursor.All(context.TODO(), &codeHosts) + if err != nil { + return nil, err + } + return codeHosts, nil +} + +func (c *CodehostColl) CodeHostList() ([]*models.CodeHost, error) { + codeHosts := make([]*models.CodeHost, 0) + cursor, err := c.Collection.Find(context.TODO(), bson.M{}) + if err != nil { + return nil, err + } + err = cursor.All(context.TODO(), &codeHosts) + if err != nil { + return nil, err + } + return codeHosts, nil +} + +func (c *CodehostColl) DeleteCodeHostByID(ID int) error { + query := bson.M{"id": ID, "deleted_at": 0} + change := bson.M{"$set": bson.M{ + "deleted_at": time.Now().Unix(), + }} + _, err := c.Collection.UpdateOne(context.TODO(), query, change) + if err != nil { + log.Errorf("repository update fail,err:%s", err) + return err + } + return nil +} + +func (c *CodehostColl) UpdateCodeHost(host *models.CodeHost) (*models.CodeHost, error) { + query := bson.M{"id": host.ID, "deleted_at": 0} + modifyValue := bson.M{ + "type": host.Type, + "address": host.Address, + "namespace": host.Namespace, + "application_id": host.ApplicationId, + "client_secret": host.ClientSecret, + "region": host.Region, + "username": host.Username, + "password": host.Password, + "enable_proxy": host.EnableProxy, + "alias": host.Alias, + "updated_at": time.Now().Unix(), + } + if host.Type == setting.SourceFromGerrit { + modifyValue["access_token"] = host.AccessToken + } else if host.Type == setting.SourceFromGitee || host.Type == setting.SourceFromGitlab { + modifyValue["access_token"] = host.AccessToken + modifyValue["refresh_token"] = host.RefreshToken + modifyValue["updated_at"] = host.UpdatedAt + } else if host.Type == setting.SourceFromOther { + modifyValue["auth_type"] = host.AuthType + modifyValue["ssh_key"] = host.SSHKey + modifyValue["private_access_token"] = host.PrivateAccessToken + } + + change := bson.M{"$set": modifyValue} + _, err := c.Collection.UpdateOne(context.TODO(), query, change) + return host, err +} + +func (c *CodehostColl) UpdateCodeHostByToken(host *models.CodeHost) (*models.CodeHost, error) { + query := bson.M{"id": host.ID, "deleted_at": 0} + change := bson.M{"$set": bson.M{ + "is_ready": "2", + "access_token": host.AccessToken, + "updated_at": time.Now().Unix(), + "refresh_token": host.RefreshToken, + }} + _, err := c.Collection.UpdateOne(context.TODO(), query, change) + return host, err +} diff --git a/pkg/microservice/systemconfig/core/codehost/service/codehost.go b/pkg/microservice/systemconfig/core/codehost/service/codehost.go new file mode 100644 index 0000000000000000000000000000000000000000..df94619521218d646be33c3335ce568cf7c3f984 --- /dev/null +++ b/pkg/microservice/systemconfig/core/codehost/service/codehost.go @@ -0,0 +1,273 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "time" + + "go.uber.org/zap" + "golang.org/x/oauth2" + + "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/systemconfig/core/codehost/internal/oauth" + "github.com/koderover/zadig/pkg/microservice/systemconfig/core/codehost/repository/models" + "github.com/koderover/zadig/pkg/microservice/systemconfig/core/codehost/repository/mongodb" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/client/aslan" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" + "github.com/koderover/zadig/pkg/tool/crypto" +) + +const callback = "/api/directory/codehosts/callback" + +func CreateCodeHost(codehost *models.CodeHost, _ *zap.SugaredLogger) (*models.CodeHost, error) { + if codehost.Type == setting.SourceFromCodeHub || codehost.Type == setting.SourceFromOther { + codehost.IsReady = "2" + } + if codehost.Type == setting.SourceFromGerrit { + codehost.IsReady = "2" + codehost.AccessToken = base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", codehost.Username, codehost.Password))) + } + + if codehost.Alias != "" { + if _, err := mongodb.NewCodehostColl().GetCodeHostByAlias(codehost.Alias); err == nil { + return nil, fmt.Errorf("alias cannot have the same name") + } + } + + codehost.CreatedAt = time.Now().Unix() + codehost.UpdatedAt = time.Now().Unix() + + list, err := mongodb.NewCodehostColl().CodeHostList() + if err != nil { + return nil, err + } + codehost.ID = len(list) + 1 + return mongodb.NewCodehostColl().AddCodeHost(codehost) +} + +func encypteCodeHost(encryptedKey string, codeHosts []*models.CodeHost, log *zap.SugaredLogger) ([]*models.CodeHost, error) { + aesKey, err := aslan.New(config.AslanServiceAddress()).GetTextFromEncryptedKey(encryptedKey) + if err != nil { + log.Errorf("ListCodeHost GetTextFromEncryptedKey error:%s", err) + return nil, err + } + var result []*models.CodeHost + for _, codeHost := range codeHosts { + if len(codeHost.Password) > 0 { + codeHost.Password, err = crypto.AesEncryptByKey(codeHost.Password, aesKey.PlainText) + if err != nil { + log.Errorf("ListCodeHost AesEncryptByKey error:%s", err) + return nil, err + } + } + if len(codeHost.ClientSecret) > 0 { + codeHost.ClientSecret, err = crypto.AesEncryptByKey(codeHost.ClientSecret, aesKey.PlainText) + if err != nil { + log.Errorf("ListCodeHost AesEncryptByKey error:%s", err) + return nil, err + } + } + if len(codeHost.SSHKey) > 0 { + codeHost.SSHKey, err = crypto.AesEncryptByKey(codeHost.SSHKey, aesKey.PlainText) + if err != nil { + log.Errorf("ListCodeHost AesEncryptByKey error:%s", err) + return nil, err + } + } + + if len(codeHost.PrivateAccessToken) > 0 { + codeHost.PrivateAccessToken, err = crypto.AesEncryptByKey(codeHost.PrivateAccessToken, aesKey.PlainText) + if err != nil { + log.Errorf("ListCodeHost AesEncryptByKey error:%s", err) + return nil, err + } + } + + result = append(result, codeHost) + } + return result, nil +} + +func ListInternal(address, owner, source string, _ *zap.SugaredLogger) ([]*models.CodeHost, error) { + return mongodb.NewCodehostColl().List(&mongodb.ListArgs{ + Address: address, + Owner: owner, + Source: source, + }) +} + +func List(encryptedKey, address, owner, source string, log *zap.SugaredLogger) ([]*models.CodeHost, error) { + codeHosts, err := mongodb.NewCodehostColl().List(&mongodb.ListArgs{ + Address: address, + Owner: owner, + Source: source, + }) + if err != nil { + log.Errorf("ListCodeHost error:%s", err) + return nil, err + } + return encypteCodeHost(encryptedKey, codeHosts, log) +} + +func DeleteCodeHost(id int, _ *zap.SugaredLogger) error { + return mongodb.NewCodehostColl().DeleteCodeHostByID(id) +} + +func UpdateCodeHost(host *models.CodeHost, _ *zap.SugaredLogger) (*models.CodeHost, error) { + if host.Type == setting.SourceFromGerrit { + host.AccessToken = base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", host.Username, host.Password))) + } + + var oldAlias string + oldCodeHost, err := mongodb.NewCodehostColl().GetCodeHostByID(host.ID, false) + if err == nil { + oldAlias = oldCodeHost.Alias + } + if host.Alias != "" && host.Alias != oldAlias { + if _, err := mongodb.NewCodehostColl().GetCodeHostByAlias(host.Alias); err == nil { + return nil, fmt.Errorf("alias cannot have the same name") + } + } + + return mongodb.NewCodehostColl().UpdateCodeHost(host) +} + +func UpdateCodeHostByToken(host *models.CodeHost, _ *zap.SugaredLogger) (*models.CodeHost, error) { + return mongodb.NewCodehostColl().UpdateCodeHostByToken(host) +} + +func GetCodeHost(id int, ignoreDelete bool, _ *zap.SugaredLogger) (*models.CodeHost, error) { + return mongodb.NewCodehostColl().GetCodeHostByID(id, ignoreDelete) +} + +type state struct { + CodeHostID int `json:"code_host_id"` + RedirectURL string `json:"redirect_url"` +} + +func AuthCodeHost(redirectURI string, codeHostID int, logger *zap.SugaredLogger) (string, error) { + codeHost, err := GetCodeHost(codeHostID, false, logger) + if err != nil { + logger.Errorf("GetCodeHost:%d err:%s", codeHostID, err) + return "", err + } + redirectParsedURL, err := url.Parse(redirectURI) + if err != nil { + logger.Errorf("Parse redirectURI:%s err:%s", redirectURI, err) + return "", err + } + redirectParsedURL.Path = callback + oauth, err := newOAuth(codeHost.Type, redirectParsedURL.String(), codeHost.ApplicationId, codeHost.ClientSecret, codeHost.Address) + if err != nil { + logger.Errorf("NewOAuth:%s err:%s", codeHost.Type, err) + return "", err + } + stateStruct := state{ + CodeHostID: codeHost.ID, + RedirectURL: redirectURI, + } + bs, err := json.Marshal(stateStruct) + if err != nil { + logger.Errorf("Marshal err:%s", err) + return "", err + } + return oauth.LoginURL(base64.URLEncoding.EncodeToString(bs)), nil +} + +func HandleCallback(stateStr string, r *http.Request, logger *zap.SugaredLogger) (string, error) { + // TODO：validate the code + // https://www.jianshu.com/p/c7c8f51713b6 + decryptedState, err := base64.URLEncoding.DecodeString(stateStr) + if err != nil { + logger.Errorf("DecodeString err:%s", err) + return "", err + } + var sta state + if err := json.Unmarshal(decryptedState, &sta); err != nil { + logger.Errorf("Unmarshal err:%s", err) + return "", err + } + redirectParsedURL, err := url.Parse(sta.RedirectURL) + if err != nil { + logger.Errorf("ParseURL:%s err:%s", sta.RedirectURL, err) + return "", err + } + codehost, err := GetCodeHost(sta.CodeHostID, false, logger) + if err != nil { + return handle(redirectParsedURL, err) + } + callbackURL := url.URL{ + Scheme: redirectParsedURL.Scheme, + Host: redirectParsedURL.Host, + Path: callback, + } + o, err := newOAuth(codehost.Type, callbackURL.String(), codehost.ApplicationId, codehost.ClientSecret, codehost.Address) + if err != nil { + logger.Errorf("newOAuth err:%s", err) + return handle(redirectParsedURL, err) + } + token, err := o.HandleCallback(r, codehost) + if err != nil { + logger.Errorf("HandleCallback err:%s", err) + return handle(redirectParsedURL, err) + } + codehost.AccessToken = token.AccessToken + codehost.RefreshToken = token.RefreshToken + if _, err := UpdateCodeHostByToken(codehost, logger); err != nil { + logger.Errorf("UpdateCodeHostByToken err:%s", err) + return handle(redirectParsedURL, err) + } + logger.Infof("success update codehost ready status") + return handle(redirectParsedURL, nil) +} + +func newOAuth(provider, callbackURL, clientID, clientSecret, address string) (*oauth.OAuth, error) { + switch provider { + case systemconfig.GitHubProvider: + return oauth.New(callbackURL, clientID, clientSecret, []string{"repo", "user"}, oauth2.Endpoint{ + AuthURL: address + "/login/oauth/authorize", + TokenURL: address + "/login/oauth/access_token", + }), nil + case systemconfig.GitLabProvider: + return oauth.New(callbackURL, clientID, clientSecret, []string{"api", "read_user"}, oauth2.Endpoint{ + AuthURL: address + "/oauth/authorize", + TokenURL: address + "/oauth/token", + }), nil + case systemconfig.GiteeProvider: + return oauth.New(callbackURL, clientID, clientSecret, []string{"projects", "pull_requests", "hook", "groups"}, oauth2.Endpoint{ + AuthURL: address + "/oauth/authorize", + TokenURL: address + "/oauth/token", + }), nil + } + return nil, errors.New("illegal provider") +} + +func handle(url *url.URL, err error) (string, error) { + if err != nil { + url.Query().Add("err", err.Error()) + } else { + url.Query().Add("success", "true") + } + return url.String(), nil +} diff --git a/pkg/microservice/systemconfig/core/connector/handler/connector.go b/pkg/microservice/systemconfig/core/connector/handler/connector.go new file mode 100644 index 0000000000000000000000000000000000000000..6d5d69e6262767a611ff1837bf8232a91d7c4fe9 --- /dev/null +++ b/pkg/microservice/systemconfig/core/connector/handler/connector.go @@ -0,0 +1,85 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/systemconfig/core/connector/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +func CreateConnector(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := &service.Connector{} + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return + } + + ctx.Err = service.CreateConnector(args, ctx.Logger) +} + +func ListConnectors(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + encryptedKey := c.Query("encryptedKey") + if len(encryptedKey) == 0 { + ctx.Err = e.ErrInvalidParam + return + } + ctx.Resp, ctx.Err = service.ListConnectors(encryptedKey, ctx.Logger) +} + +func ListConnectorsInternal(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.ListConnectorsInternal(ctx.Logger) +} + +func DeleteConnector(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Err = service.DeleteConnector(c.Param("id"), ctx.Logger) +} + +func GetConnector(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.GetConnector(c.Param("id"), ctx.Logger) +} + +func UpdateConnector(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := &service.Connector{} + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return + } + args.ID = c.Param("id") + + ctx.Err = service.UpdateConnector(args, ctx.Logger) +} diff --git a/pkg/microservice/systemconfig/core/connector/handler/router.go b/pkg/microservice/systemconfig/core/connector/handler/router.go new file mode 100644 index 0000000000000000000000000000000000000000..1795bdd0ec59772e61e83e4d49fcf2f4ad43bd5c --- /dev/null +++ b/pkg/microservice/systemconfig/core/connector/handler/router.go @@ -0,0 +1,35 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" +) + +type Router struct{} + +func (*Router) Inject(router *gin.RouterGroup) { + connector := router.Group("connectors") + { + connector.POST("", CreateConnector) + connector.GET("", ListConnectors) + connector.GET("/internal", ListConnectorsInternal) + connector.GET("/:id", GetConnector) + connector.PUT("/:id", UpdateConnector) + connector.DELETE("/:id", DeleteConnector) + } +} diff --git a/pkg/microservice/systemconfig/core/connector/service/connector.go b/pkg/microservice/systemconfig/core/connector/service/connector.go new file mode 100644 index 0000000000000000000000000000000000000000..7c7fa8d5db2b28381d7e9ea69d2c9243446b41b5 --- /dev/null +++ b/pkg/microservice/systemconfig/core/connector/service/connector.go @@ -0,0 +1,176 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "encoding/json" + + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/systemconfig/core/repository/models" + "github.com/koderover/zadig/pkg/microservice/systemconfig/core/repository/orm" + "github.com/koderover/zadig/pkg/shared/client/aslan" + "github.com/koderover/zadig/pkg/tool/crypto" +) + +func ListConnectorsInternal(logger *zap.SugaredLogger) ([]*Connector, error) { + cs, err := orm.NewConnectorColl().List() + if err != nil { + logger.Errorf("Failed to list connectors, err: %s", err) + return nil, err + } + + var res []*Connector + for _, c := range cs { + cf := make(map[string]interface{}) + err = json.Unmarshal([]byte(c.Config), &cf) + if err != nil { + logger.Errorf("Failed to unmarshal config, err: %s", err) + continue + } + res = append(res, &Connector{ + ConnectorBase: ConnectorBase{ + Type: ConnectorType(c.Type), + }, + ID: c.ID, + Name: c.Name, + Config: cf, + }) + } + + return res, nil +} + +func ListConnectors(encryptedKey string, logger *zap.SugaredLogger) ([]*Connector, error) { + aesKey, err := aslan.New(config.AslanServiceAddress()).GetTextFromEncryptedKey(encryptedKey) + if err != nil { + logger.Errorf("ListConnectors GetTextFromEncryptedKey, err: %s", err) + return nil, err + } + cs, err := orm.NewConnectorColl().List() + if err != nil { + logger.Errorf("Failed to list connectors, err: %s", err) + return nil, err + } + config, err := aslan.New(config.AslanServiceAddress()).GetDefaultLogin() + if err != nil { + logger.Errorf("Failed to list connectors, err: %s", err) + return nil, err + } + var res []*Connector + for _, c := range cs { + cf := make(map[string]interface{}) + err = json.Unmarshal([]byte(c.Config), &cf) + if err != nil { + logger.Errorf("Failed to unmarshal config, err: %s", err) + continue + } + if pw, ok := cf["bindPW"]; ok { + cf["bindPW"], err = crypto.AesEncryptByKey(pw.(string), aesKey.PlainText) + if err != nil { + logger.Errorf("ListConnectors AesEncryptByKey, err: %s", err) + return nil, err + } + } + if clientSecret, ok := cf["clientSecret"]; ok { + cf["clientSecret"], err = crypto.AesEncryptByKey(clientSecret.(string), aesKey.PlainText) + if err != nil { + logger.Errorf("ListConnectors AesEncryptByKey, err: %s", err) + return nil, err + } + } + isDefault := false + if config.DefaultLogin == c.ID { + isDefault = true + } + res = append(res, &Connector{ + ConnectorBase: ConnectorBase{ + Type: ConnectorType(c.Type), + }, + ID: c.ID, + Name: c.Name, + Config: cf, + IsDefault: isDefault, + }) + } + + return res, nil +} + +func GetConnector(id string, logger *zap.SugaredLogger) (*Connector, error) { + c, err := orm.NewConnectorColl().Get(id) + if err != nil { + logger.Errorf("Failed to get connector %s, err: %s", id, err) + return nil, err + } + + cf := make(map[string]interface{}) + err = json.Unmarshal([]byte(c.Config), &cf) + if err != nil { + logger.Warnf("Failed to unmarshal config, err: %s", err) + + } + + return &Connector{ + ConnectorBase: ConnectorBase{ + Type: ConnectorType(c.Type), + }, + ID: c.ID, + Name: c.Name, + Config: cf, + }, nil + +} + +func DeleteConnector(id string, _ *zap.SugaredLogger) error { + return orm.NewConnectorColl().Delete(id) +} + +func CreateConnector(ct *Connector, logger *zap.SugaredLogger) error { + cf, err := json.Marshal(ct.Config) + if err != nil { + logger.Errorf("Failed to marshal config, err: %s", err) + return err + } + + obj := &models.Connector{ + ID: ct.ID, + Name: ct.Name, + Type: string(ct.Type), + Config: string(cf), + } + + return orm.NewConnectorColl().Create(obj) +} + +func UpdateConnector(ct *Connector, logger *zap.SugaredLogger) error { + cf, err := json.Marshal(ct.Config) + if err != nil { + logger.Errorf("Failed to marshal config, err: %s", err) + return err + } + + obj := &models.Connector{ + ID: ct.ID, + Name: ct.Name, + Type: string(ct.Type), + Config: string(cf), + } + + return orm.NewConnectorColl().Update(obj) +} diff --git a/pkg/microservice/systemconfig/core/connector/service/types.go b/pkg/microservice/systemconfig/core/connector/service/types.go new file mode 100644 index 0000000000000000000000000000000000000000..2696bc12ea6a6a106c59e44d1b04a449ebb50ebd --- /dev/null +++ b/pkg/microservice/systemconfig/core/connector/service/types.go @@ -0,0 +1,85 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "encoding/json" + + "github.com/dexidp/dex/connector/gitea" + "github.com/dexidp/dex/connector/github" + "github.com/dexidp/dex/connector/gitlab" + "github.com/dexidp/dex/connector/google" + "github.com/dexidp/dex/connector/ldap" + "github.com/dexidp/dex/connector/linkedin" + "github.com/dexidp/dex/connector/microsoft" + "github.com/dexidp/dex/connector/oidc" +) + +type ConnectorType string + +const ( + TypeLDAP ConnectorType = "ldap" + TypeGitHub ConnectorType = "github" + TypeGitlab ConnectorType = "gitlab" + TypeOIDC ConnectorType = "oidc" + TypeGitea ConnectorType = "gitea" + TypeGoogle ConnectorType = "google" + TypeLinkedIn ConnectorType = "linkedin" + TypeMicrosoft ConnectorType = "microsoft" +) + +type Connector struct { + ConnectorBase + IsDefault bool `json:"is_default"` + ID string `json:"id"` + Name string `json:"name"` + Config interface{} `json:"config"` +} + +type ConnectorBase struct { + Type ConnectorType `json:"type"` +} + +func (c *Connector) UnmarshalJSON(data []byte) error { + cb := &ConnectorBase{} + if err := json.Unmarshal(data, cb); err != nil { + return err + } + + switch cb.Type { + case TypeLDAP: + c.Config = &ldap.Config{} + case TypeGitHub: + c.Config = &github.Config{} + case TypeGitlab: + c.Config = &gitlab.Config{} + case TypeOIDC: + c.Config = &oidc.Config{} + case TypeGitea: + c.Config = &gitea.Config{} + case TypeGoogle: + c.Config = &google.Config{} + case TypeLinkedIn: + c.Config = &linkedin.Config{} + case TypeMicrosoft: + c.Config = µsoft.Config{} + } + + type tmp Connector + + return json.Unmarshal(data, (*tmp)(c)) +} diff --git a/pkg/microservice/systemconfig/core/email/handler/email.go b/pkg/microservice/systemconfig/core/email/handler/email.go new file mode 100644 index 0000000000000000000000000000000000000000..1f3d714d67e791d1ded0a330943a306d347aa33b --- /dev/null +++ b/pkg/microservice/systemconfig/core/email/handler/email.go @@ -0,0 +1,112 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/systemconfig/core/email/repository/models" + "github.com/koderover/zadig/pkg/microservice/systemconfig/core/email/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +func GetEmailHost(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + ctx.Resp, ctx.Err = service.GetEmailHost(ctx.Logger) +} + +func GetEmailHostInternal(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + ctx.Resp, ctx.Err = service.GetEmailHostInternal(ctx.Logger) +} + +func GetEncryptedEmailHost(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + encryptedKey := c.Query("encryptedKey") + if len(encryptedKey) == 0 { + ctx.Err = e.ErrInvalidParam + return + } + ctx.Resp, ctx.Err = service.GetEncryptedEmailHost(encryptedKey, ctx.Logger) +} + +func CreateEmailHost(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + req := new(models.EmailHost) + if err := c.ShouldBindJSON(req); err != nil { + ctx.Err = err + return + } + ctx.Resp, ctx.Err = service.CreateEmailHost(req, ctx.Logger) +} + +func UpdateEmailHost(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + req := new(models.EmailHost) + if err := c.ShouldBindJSON(req); err != nil { + ctx.Err = err + return + } + ctx.Resp, ctx.Err = service.UpdateEmailHost(req, ctx.Logger) +} + +func DeleteEmailHost(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + ctx.Err = service.DeleteEmailHost(ctx.Logger) +} + +func GetEmailService(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + ctx.Resp, ctx.Err = service.GetEmailService(ctx.Logger) +} + +func CreateEmailService(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + req := new(models.EmailService) + if err := c.ShouldBindJSON(req); err != nil { + ctx.Err = err + return + } + ctx.Resp, ctx.Err = service.CreateEmailService(req, ctx.Logger) +} + +func UpdateEmailService(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + req := new(models.EmailService) + if err := c.ShouldBindJSON(req); err != nil { + ctx.Err = err + return + } + ctx.Resp, ctx.Err = service.UpdateEmailService(req, ctx.Logger) +} + +func DeleteEmailService(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + ctx.Err = service.DeleteEmailService(ctx.Logger) +} diff --git a/pkg/microservice/systemconfig/core/email/handler/router.go b/pkg/microservice/systemconfig/core/email/handler/router.go new file mode 100644 index 0000000000000000000000000000000000000000..aea6d9b222dffd3521b111af3570a46c4a661389 --- /dev/null +++ b/pkg/microservice/systemconfig/core/email/handler/router.go @@ -0,0 +1,41 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" +) + +type Router struct{} + +func (*Router) Inject(router *gin.RouterGroup) { + emails := router.Group("emails") + { + emails.GET("/host", GetEmailHost) + emails.POST("/host", CreateEmailHost) + emails.PATCH("/host", UpdateEmailHost) + emails.DELETE("/host", DeleteEmailHost) + + emails.GET("/internal/host", GetEncryptedEmailHost) + emails.GET("/host/internal", GetEmailHostInternal) + + emails.GET("/service", GetEmailService) + emails.POST("/service", CreateEmailService) + emails.PATCH("/service", UpdateEmailService) + emails.DELETE("/service", DeleteEmailService) + } +} diff --git a/pkg/microservice/systemconfig/core/email/repository/models/emailhost.go b/pkg/microservice/systemconfig/core/email/repository/models/emailhost.go new file mode 100644 index 0000000000000000000000000000000000000000..1b1b493e9b30da80346138ede958590e86b6dfd8 --- /dev/null +++ b/pkg/microservice/systemconfig/core/email/repository/models/emailhost.go @@ -0,0 +1,32 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +type EmailHost struct { + Name string `bson:"name" json:"name"` + Port int `bson:"port" json:"port"` + Username string `bson:"username" json:"username"` + Password string `bson:"password" json:"password"` + IsTLS bool `bson:"is_tls" json:"isTLS"` + CreatedAt int64 `bson:"created_at" json:"created_at"` + DeletedAt int64 `bson:"deleted_at" json:"deleted_at"` + UpdatedAt int64 `bson:"updated_at" json:"updated_at"` +} + +func (EmailHost) TableName() string { + return "email_host" +} diff --git a/pkg/microservice/systemconfig/core/email/repository/models/emailservice.go b/pkg/microservice/systemconfig/core/email/repository/models/emailservice.go new file mode 100644 index 0000000000000000000000000000000000000000..37c60e33092995ef1910e81ce66a3362ab91cb74 --- /dev/null +++ b/pkg/microservice/systemconfig/core/email/repository/models/emailservice.go @@ -0,0 +1,31 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +type EmailService struct { + Name string `json:"name" bson:"name"` + Address string `json:"address" bson:"address"` + DisplayName string `json:"display_name" bson:"display_name"` + Theme string `json:"theme" bson:"theme"` + CreatedAt int64 `json:"created_at" bson:"created_at"` + UpdatedAt int64 `json:"updated_at" bson:"updated_at"` + DeletedAt int64 `json:"deleted_at" bson:"deleted_at"` +} + +func (EmailService) TableName() string { + return "email_service" +} diff --git a/pkg/microservice/systemconfig/core/email/repository/mongodb/emailhost.go b/pkg/microservice/systemconfig/core/email/repository/mongodb/emailhost.go new file mode 100644 index 0000000000000000000000000000000000000000..baa3d3f93d6a88380f02a4d9068722638e06438f --- /dev/null +++ b/pkg/microservice/systemconfig/core/email/repository/mongodb/emailhost.go @@ -0,0 +1,121 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "errors" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + + "github.com/koderover/zadig/pkg/microservice/systemconfig/config" + "github.com/koderover/zadig/pkg/microservice/systemconfig/core/email/repository/models" + "github.com/koderover/zadig/pkg/tool/log" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type EmailHostColl struct { + *mongo.Collection + + coll string +} + +func NewEmailHostColl() *EmailHostColl { + name := models.EmailHost{}.TableName() + coll := &EmailHostColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} + + return coll +} + +func (c *EmailHostColl) GetCollectionName() string { + return c.coll +} +func (c *EmailHostColl) EnsureIndex(ctx context.Context) error { + return nil +} + +func (c *EmailHostColl) Find() (*models.EmailHost, error) { + emailHost := new(models.EmailHost) + query := bson.M{"deleted_at": 0} + + ctx := context.Background() + + err := c.Collection.FindOne(ctx, query).Decode(emailHost) + if err != nil { + return nil, nil + } + return emailHost, nil +} + +func (c *EmailHostColl) Update(emailHost *models.EmailHost) (*models.EmailHost, error) { + query := bson.M{"deleted_at": 0} + change := bson.M{"$set": bson.M{ + "name": emailHost.Name, + "port": emailHost.Port, + "username": emailHost.Username, + "password": emailHost.Password, + "is_tls": emailHost.IsTLS, + "updated_at": time.Now().Unix(), + }} + + _, err := c.Collection.UpdateOne(context.TODO(), query, change) + if err != nil { + log.Error("repository Update EmailHostColl err : %v", err) + return nil, err + } + return emailHost, nil +} + +func (c *EmailHostColl) Delete() error { + query := bson.M{"deleted_at": 0} + change := bson.M{"$set": bson.M{ + "deleted_at": time.Now().Unix(), + }} + + _, err := c.Collection.UpdateOne(context.TODO(), query, change) + if err != nil { + log.Error("repository Delete EmailHostColl err : %v", err) + return err + } + return nil +} + +func (c *EmailHostColl) Add(emailHost *models.EmailHost) (*models.EmailHost, error) { + // TODO: tmp solution to avoid bug + var res []*models.EmailHost + query := bson.M{"deleted_at": 0} + cur, err := c.Collection.Find(context.TODO(), query) + if err != nil { + return nil, err + } + err = cur.All(context.TODO(), &res) + if err != nil { + return nil, err + } + if len(res) >= 1 { + return nil, errors.New("cant add more than one emailhost") + } + + _, err = c.Collection.InsertOne(context.TODO(), emailHost) + if err != nil { + log.Error("repository AddEmailHost err : %v", err) + return nil, err + } + return emailHost, nil +} diff --git a/pkg/microservice/systemconfig/core/email/repository/mongodb/emailservice.go b/pkg/microservice/systemconfig/core/email/repository/mongodb/emailservice.go new file mode 100644 index 0000000000000000000000000000000000000000..4a49a426324857c203555b280bfa44521dbedae1 --- /dev/null +++ b/pkg/microservice/systemconfig/core/email/repository/mongodb/emailservice.go @@ -0,0 +1,116 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "errors" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + + "github.com/koderover/zadig/pkg/microservice/systemconfig/config" + "github.com/koderover/zadig/pkg/microservice/systemconfig/core/email/repository/models" + "github.com/koderover/zadig/pkg/tool/log" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type EmailServiceColl struct { + *mongo.Collection + + coll string +} + +func NewEmailServiceColl() *EmailServiceColl { + name := models.EmailService{}.TableName() + coll := &EmailServiceColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} + + return coll +} + +func (c *EmailServiceColl) GetCollectionName() string { + return c.coll +} +func (c *EmailServiceColl) EnsureIndex(ctx context.Context) error { + return nil +} + +func (c *EmailServiceColl) AddEmailService(iEmailService *models.EmailService) (*models.EmailService, error) { + // TODO: tmp solution to avoid bug + var res []*models.EmailService + query := bson.M{"deleted_at": 0} + cur, err := c.Collection.Find(context.TODO(), query) + if err != nil { + return nil, err + } + err = cur.All(context.TODO(), &res) + if err != nil { + return nil, err + } + if len(res) >= 1 { + return nil, errors.New("cant add more than one emailservice") + } + + _, err = c.Collection.InsertOne(context.TODO(), iEmailService) + if err != nil { + log.Error("repository AddEmailService err : %v", err) + return nil, err + } + return iEmailService, nil +} + +func (c *EmailServiceColl) GetEmailService() (*models.EmailService, error) { + query := bson.M{"deleted_at": 0} + iEmailService := &models.EmailService{} + err := c.Collection.FindOne(context.TODO(), query).Decode(iEmailService) + if err != nil { + return nil, nil + } + return iEmailService, nil +} + +func (c *EmailServiceColl) UpdateEmailService(iEmailService *models.EmailService) (*models.EmailService, error) { + query := bson.M{"deleted_at": 0} + change := bson.M{"$set": bson.M{ + "name": iEmailService.Name, + "address": iEmailService.Address, + "display_name": iEmailService.DisplayName, + "theme": iEmailService.Theme, + "updated_at": time.Now().Unix(), + }} + + _, err := c.Collection.UpdateOne(context.TODO(), query, change) + if err != nil { + log.Error("repository UpdateEmailService err : %v", err) + return nil, err + } + return iEmailService, nil +} + +func (c *EmailServiceColl) DeleteEmailService() error { + query := bson.M{"deleted_at": 0} + change := bson.M{"$set": bson.M{ + "deleted_at": time.Now().Unix(), + }} + _, err := c.Collection.UpdateOne(context.TODO(), query, change) + if err != nil { + log.Error("repository DeleteEmailServiceByOrgID err : %v", err) + return err + } + return nil +} diff --git a/pkg/microservice/systemconfig/core/email/service/email.go b/pkg/microservice/systemconfig/core/email/service/email.go new file mode 100644 index 0000000000000000000000000000000000000000..f191e7651d384624d03488ce3cce308faf820fcb --- /dev/null +++ b/pkg/microservice/systemconfig/core/email/service/email.go @@ -0,0 +1,98 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "time" + + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/systemconfig/core/email/repository/models" + "github.com/koderover/zadig/pkg/microservice/systemconfig/core/email/repository/mongodb" + "github.com/koderover/zadig/pkg/shared/client/aslan" + "github.com/koderover/zadig/pkg/tool/crypto" +) + +func GetEmailHost(_ *zap.SugaredLogger) (*models.EmailHost, error) { + host, err := mongodb.NewEmailHostColl().Find() + if host != nil { + host.Password = "***" + } + return host, err +} + +func GetEmailHostInternal(_ *zap.SugaredLogger) (*models.EmailHost, error) { + return mongodb.NewEmailHostColl().Find() +} + +func GetEncryptedEmailHost(encryptedKey string, log *zap.SugaredLogger) (*models.EmailHost, error) { + aesKey, err := aslan.New(config.AslanServiceAddress()).GetTextFromEncryptedKey(encryptedKey) + if err != nil { + log.Errorf("GetEncryptedEmailHost GetTextFromEncryptedKey error:%s", err) + return nil, err + } + result, err := mongodb.NewEmailHostColl().Find() + if err != nil { + log.Errorf("GetEncryptedEmailHost find email host error:%s", err) + return nil, err + } + if result == nil { + return nil, nil + } + password, err := crypto.AesEncryptByKey(result.Password, aesKey.PlainText) + if err != nil { + log.Errorf("GetEncryptedEmailHost AesEncryptByKey error:%s", err) + return nil, err + } + result.Password = password + return result, nil +} + +func CreateEmailHost(emailHost *models.EmailHost, _ *zap.SugaredLogger) (*models.EmailHost, error) { + emailHost.CreatedAt = time.Now().Unix() + emailHost.UpdatedAt = time.Now().Unix() + return mongodb.NewEmailHostColl().Add(emailHost) +} + +func UpdateEmailHost(emailHost *models.EmailHost, _ *zap.SugaredLogger) (*models.EmailHost, error) { + emailHost.UpdatedAt = time.Now().Unix() + return mongodb.NewEmailHostColl().Update(emailHost) +} + +func DeleteEmailHost(_ *zap.SugaredLogger) error { + return mongodb.NewEmailHostColl().Delete() +} + +func GetEmailService(_ *zap.SugaredLogger) (*models.EmailService, error) { + return mongodb.NewEmailServiceColl().GetEmailService() +} + +func CreateEmailService(emailService *models.EmailService, _ *zap.SugaredLogger) (*models.EmailService, error) { + emailService.CreatedAt = time.Now().Unix() + emailService.UpdatedAt = time.Now().Unix() + return mongodb.NewEmailServiceColl().AddEmailService(emailService) +} + +func UpdateEmailService(emailService *models.EmailService, _ *zap.SugaredLogger) (*models.EmailService, error) { + emailService.UpdatedAt = time.Now().Unix() + return mongodb.NewEmailServiceColl().UpdateEmailService(emailService) +} + +func DeleteEmailService(_ *zap.SugaredLogger) error { + return mongodb.NewEmailServiceColl().DeleteEmailService() +} diff --git a/pkg/microservice/systemconfig/core/features/handler/features.go b/pkg/microservice/systemconfig/core/features/handler/features.go new file mode 100644 index 0000000000000000000000000000000000000000..389ecfb855056bc002265d77e8a775a73c654c7a --- /dev/null +++ b/pkg/microservice/systemconfig/core/features/handler/features.go @@ -0,0 +1,54 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/systemconfig/core/features/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" +) + +type feature struct { + Name string `json:"name"` + Enabled bool `json:"enabled"` +} + +func GetFeature(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + name := c.Param("name") + enabled := service.Features.FeatureEnabled(service.Feature(name)) + + ctx.Resp = &feature{ + Name: c.Param("name"), + Enabled: enabled, + } +} + +func UpdateOrCreateFeature(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + req := new(service.FeatureReq) + if err := c.ShouldBindJSON(req); err != nil { + ctx.Err = err + return + } + ctx.Err = service.UpdateOrCreateFeature(req) +} diff --git a/pkg/microservice/aslan/core/setting/handler/router.go b/pkg/microservice/systemconfig/core/features/handler/router.go similarity index 81% rename from pkg/microservice/aslan/core/setting/handler/router.go rename to pkg/microservice/systemconfig/core/features/handler/router.go index 2b4eef2e9f36dbdabba0a1857ea5aede9432f4a9..7e3a8e3476fc95671b4cffaeb2b8ad069f6525c3 100644 --- a/pkg/microservice/aslan/core/setting/handler/router.go +++ b/pkg/microservice/systemconfig/core/features/handler/router.go @@ -18,17 +18,14 @@ package handler import ( "github.com/gin-gonic/gin" - - gin2 "github.com/koderover/zadig/pkg/middleware/gin" ) type Router struct{} func (*Router) Inject(router *gin.RouterGroup) { - router.Use(gin2.Auth()) - - user := router.Group("user") + features := router.Group("features") { - user.GET("/kube/config", GetUserKubeConfig) + features.PUT("/:name", UpdateOrCreateFeature) + features.GET("/:name", GetFeature) } } diff --git a/pkg/microservice/systemconfig/core/features/repository/models/feature.go b/pkg/microservice/systemconfig/core/features/repository/models/feature.go new file mode 100644 index 0000000000000000000000000000000000000000..63fe2f63bba8ed22a90dcbbaf179ce6d766e2ab8 --- /dev/null +++ b/pkg/microservice/systemconfig/core/features/repository/models/feature.go @@ -0,0 +1,26 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +type Feature struct { + Name string `bson:"name" json:"name"` + Enabled bool `bson:"enabled" json:"enabled"` +} + +func (Feature) TableName() string { + return "feature" +} diff --git a/pkg/microservice/systemconfig/core/features/repository/mongodb/feature.go b/pkg/microservice/systemconfig/core/features/repository/mongodb/feature.go new file mode 100644 index 0000000000000000000000000000000000000000..2b32687d593b71d6ed5fa6609077a46691f3322a --- /dev/null +++ b/pkg/microservice/systemconfig/core/features/repository/mongodb/feature.go @@ -0,0 +1,65 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/microservice/systemconfig/config" + "github.com/koderover/zadig/pkg/microservice/systemconfig/core/features/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type FeatureColl struct { + *mongo.Collection + + coll string +} + +func NewFeatureColl() *FeatureColl { + name := models.Feature{}.TableName() + coll := &FeatureColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} + + return coll +} + +func (c *FeatureColl) GetCollectionName() string { + return c.coll +} + +func (c *FeatureColl) UpdateOrCreateFeature(ctx context.Context, obj *models.Feature) error { + q := bson.M{"name": obj.Name} + opts := options.Replace().SetUpsert(true) + _, err := c.ReplaceOne(ctx, q, obj, opts) + return err +} + +func (c *FeatureColl) ListFeatures() ([]*models.Feature, error) { + var fs []*models.Feature + cursor, err := c.Find(context.TODO(), bson.M{}) + if err != nil { + return nil, err + } + if err := cursor.All(context.TODO(), &fs); err != nil { + return nil, err + } + return fs, nil +} diff --git a/pkg/microservice/systemconfig/core/features/service/feature.go b/pkg/microservice/systemconfig/core/features/service/feature.go new file mode 100644 index 0000000000000000000000000000000000000000..4b91fd7a5f34112d04f5aaf296d9a42e3f16fd42 --- /dev/null +++ b/pkg/microservice/systemconfig/core/features/service/feature.go @@ -0,0 +1,122 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "context" + "strconv" + "strings" + + "github.com/koderover/zadig/pkg/microservice/systemconfig/core/features/repository/models" + "github.com/koderover/zadig/pkg/microservice/systemconfig/core/features/repository/mongodb" + "github.com/koderover/zadig/pkg/tool/log" +) + +type Feature string + +const ( + ModernWorkflow Feature = "ModernWorkflow" + CommunityProjectRepository Feature = "CommunityProjectRepository" + UserRegistration Feature = "UserRegistration" +) + +type FeatureGates map[Feature]bool + +var Features FeatureGates = map[Feature]bool{ + ModernWorkflow: false, + CommunityProjectRepository: false, + UserRegistration: true, +} + +func (fg FeatureGates) EnabledFeatures() []Feature { + var res []Feature + + for k, v := range fg { + if v { + res = append(res, k) + } + } + + return res +} + +func (fg FeatureGates) FeatureEnabled(f Feature) bool { + return fg[f] +} + +// MergeFeatureGates merge feature config from different source +// latter feature configs will be overridden by former ones +func (fg FeatureGates) MergeFeatureGates(fs ...FeatureGates) { + for _, v := range fs { + for k, vv := range v { + fg[k] = vv + } + } +} + +func DBToFeatureGates() (FeatureGates, error) { + fg := make(FeatureGates) + fs, err := mongodb.NewFeatureColl().ListFeatures() + if err != nil { + log.Errorf("list features err:%s", err) + return nil, err + } + + for _, v := range fs { + fg[Feature(v.Name)] = v.Enabled + } + return fg, nil +} + +func FlagToFeatureGates(s string) (FeatureGates, error) { + res := make(FeatureGates) + + fs := strings.Split(s, ",") + for _, f := range fs { + kv := strings.Split(f, "=") + if len(kv) != 2 { + continue + } + boolValue, err := strconv.ParseBool(kv[1]) + if err != nil { + log.Errorf("invalid value of %s=%s, err: %v", kv[0], kv[1], err) + return nil, err + } + res[Feature(kv[0])] = boolValue + } + + return res, nil +} + +type FeatureReq struct { + Name string `json:"name"` + Enabled bool `json:"enabled"` +} + +func UpdateOrCreateFeature(req *FeatureReq) error { + if err := mongodb.NewFeatureColl().UpdateOrCreateFeature(context.TODO(), &models.Feature{ + Name: req.Name, + Enabled: req.Enabled, + }); err != nil { + return err + } + fg := FeatureGates{ + Feature(req.Name): req.Enabled, + } + Features.MergeFeatureGates(fg) + return nil +} diff --git a/pkg/microservice/systemconfig/core/jira/handler/jira.go b/pkg/microservice/systemconfig/core/jira/handler/jira.go new file mode 100644 index 0000000000000000000000000000000000000000..b97b26d4b8e41d74f1bbef7f496c3b56c73d0905 --- /dev/null +++ b/pkg/microservice/systemconfig/core/jira/handler/jira.go @@ -0,0 +1,71 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/systemconfig/core/jira/repository/models" + "github.com/koderover/zadig/pkg/microservice/systemconfig/core/jira/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +func DeleteJira(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + ctx.Err = service.DeleteJira(ctx.Logger) +} + +func GetJira(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + encryptedKey := c.Query("encryptedKey") + if len(encryptedKey) == 0 { + ctx.Err = e.ErrInvalidParam + return + } + ctx.Resp, ctx.Err = service.GeJira(encryptedKey, ctx.Logger) +} + +func GetJiraInternal(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + ctx.Resp, ctx.Err = service.GeJiraInternal(ctx.Logger) +} + +func CreateJira(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + req := new(models.Jira) + if err := c.ShouldBindJSON(req); err != nil { + ctx.Err = err + return + } + ctx.Resp, ctx.Err = service.CreateJira(req, ctx.Logger) +} + +func UpdateJira(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + req := new(models.Jira) + if err := c.ShouldBindJSON(req); err != nil { + ctx.Err = err + return + } + ctx.Resp, ctx.Err = service.UpdateJira(req, ctx.Logger) +} diff --git a/pkg/microservice/systemconfig/core/jira/handler/router.go b/pkg/microservice/systemconfig/core/jira/handler/router.go new file mode 100644 index 0000000000000000000000000000000000000000..6086766e68177af44ba0d90b0784b718eeb92b65 --- /dev/null +++ b/pkg/microservice/systemconfig/core/jira/handler/router.go @@ -0,0 +1,35 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" +) + +type Router struct{} + +func (*Router) Inject(router *gin.RouterGroup) { + jira := router.Group("jira") + { + jira.GET("", GetJira) + jira.GET("/internal", GetJiraInternal) + jira.POST("", CreateJira) + jira.PATCH("", UpdateJira) + jira.DELETE("", DeleteJira) + } + +} diff --git a/pkg/microservice/systemconfig/core/jira/repository/models/jira.go b/pkg/microservice/systemconfig/core/jira/repository/models/jira.go new file mode 100644 index 0000000000000000000000000000000000000000..e2b6d483bc747b3253ba58b5f76c44e16c7625fb --- /dev/null +++ b/pkg/microservice/systemconfig/core/jira/repository/models/jira.go @@ -0,0 +1,30 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +type Jira struct { + Host string `bson:"host" json:"host"` + User string `bson:"user" json:"user"` + AccessToken string `bson:"access_token" json:"access_token"` + CreatedAt int64 `bson:"created_at" json:"created_at"` + UpdatedAt int64 `bson:"updated_at" json:"updated_at"` + DeletedAt int64 `bson:"deleted_at" json:"deleted_at"` +} + +func (Jira) TableName() string { + return "jira" +} diff --git a/pkg/microservice/systemconfig/core/jira/repository/mongodb/jira.go b/pkg/microservice/systemconfig/core/jira/repository/mongodb/jira.go new file mode 100644 index 0000000000000000000000000000000000000000..7b544bcaf21e0cf7b48e4fc7ca96377e04c8a8a8 --- /dev/null +++ b/pkg/microservice/systemconfig/core/jira/repository/mongodb/jira.go @@ -0,0 +1,103 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + + "github.com/koderover/zadig/pkg/microservice/systemconfig/config" + "github.com/koderover/zadig/pkg/microservice/systemconfig/core/jira/repository/models" + "github.com/koderover/zadig/pkg/tool/log" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type JiraColl struct { + *mongo.Collection + + coll string +} + +func NewJiraColl() *JiraColl { + name := models.Jira{}.TableName() + coll := &JiraColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} + + return coll +} + +func (c *JiraColl) GetCollectionName() string { + return c.coll +} +func (c *JiraColl) EnsureIndex(ctx context.Context) error { + return nil +} + +func (c *JiraColl) AddJira(iJira *models.Jira) (*models.Jira, error) { + _, err := c.Collection.InsertOne(context.TODO(), iJira) + if err != nil { + log.Error("repository AddJira err : %v", err) + return nil, err + } + return iJira, nil +} + +func (c *JiraColl) UpdateJira(iJira *models.Jira) (*models.Jira, error) { + + query := bson.M{"deleted_at": 0} + change := bson.M{"$set": bson.M{ + "host": iJira.Host, + "user": iJira.User, + "access_token": iJira.AccessToken, + "updated_at": time.Now().Unix(), + }} + + _, err := c.Collection.UpdateOne(context.TODO(), query, change) + if err != nil { + log.Error("repository UpdateJira err : %v", err) + return nil, err + } + return iJira, nil +} + +func (c *JiraColl) DeleteJira() error { + + query := bson.M{"deleted_at": 0} + change := bson.M{"$set": bson.M{ + "deleted_at": time.Now().Unix(), + }} + + _, err := c.Collection.UpdateOne(context.TODO(), query, change) + if err != nil { + log.Error("repository DeleteJira err : %v", err) + return err + } + return nil +} + +func (c *JiraColl) GetJira() (*models.Jira, error) { + jira := &models.Jira{} + query := bson.M{"deleted_at": 0} + + err := c.Collection.FindOne(context.TODO(), query).Decode(jira) + if err != nil { + return nil, nil + } + return jira, nil +} diff --git a/pkg/microservice/systemconfig/core/jira/service/jira.go b/pkg/microservice/systemconfig/core/jira/service/jira.go new file mode 100644 index 0000000000000000000000000000000000000000..2b999406d7c104963fb5c3df4842ff6701acbf09 --- /dev/null +++ b/pkg/microservice/systemconfig/core/jira/service/jira.go @@ -0,0 +1,70 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "time" + + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/systemconfig/core/jira/repository/models" + "github.com/koderover/zadig/pkg/microservice/systemconfig/core/jira/repository/mongodb" + "github.com/koderover/zadig/pkg/shared/client/aslan" + "github.com/koderover/zadig/pkg/tool/crypto" +) + +func GeJira(encryptedKey string, log *zap.SugaredLogger) (*models.Jira, error) { + jira, err := mongodb.NewJiraColl().GetJira() + if err != nil { + log.Errorf("GeJira error:%s", err) + return nil, err + } + if jira == nil { + return nil, nil + } + aesKey, err := aslan.New(config.AslanServiceAddress()).GetTextFromEncryptedKey(encryptedKey) + if err != nil { + log.Errorf("GeJira GetTextFromEncryptedKey erorr:%s", err) + return nil, err + } + jira.AccessToken, err = crypto.AesEncryptByKey(jira.AccessToken, aesKey.PlainText) + if err != nil { + log.Errorf("GeJira AesEncryptByKey erorr:%s", err) + return nil, err + } + return jira, nil +} + +func GeJiraInternal(_ *zap.SugaredLogger) (*models.Jira, error) { + return mongodb.NewJiraColl().GetJira() +} + +func CreateJira(jira *models.Jira, _ *zap.SugaredLogger) (*models.Jira, error) { + jira.CreatedAt = time.Now().Unix() + jira.UpdatedAt = time.Now().Unix() + return mongodb.NewJiraColl().AddJira(jira) +} + +func UpdateJira(jira *models.Jira, _ *zap.SugaredLogger) (*models.Jira, error) { + jira.UpdatedAt = time.Now().Unix() + return mongodb.NewJiraColl().UpdateJira(jira) +} + +func DeleteJira(_ *zap.SugaredLogger) error { + return mongodb.NewJiraColl().DeleteJira() +} diff --git a/pkg/microservice/systemconfig/core/repository/models/connector.go b/pkg/microservice/systemconfig/core/repository/models/connector.go new file mode 100644 index 0000000000000000000000000000000000000000..42706039aada46e79989bc7588db1105b9eeae3f --- /dev/null +++ b/pkg/microservice/systemconfig/core/repository/models/connector.go @@ -0,0 +1,37 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import "strconv" + +type Connector struct { + ID string `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + ResourceVersion string `json:"resource_version"` + Config string `json:"config"` +} + +func (m *Connector) IncreaseResourceVersion() { + rv, _ := strconv.Atoi(m.ResourceVersion) + m.ResourceVersion = strconv.Itoa(rv + 1) +} + +// TableName sets the insert table name for this struct type +func (Connector) TableName() string { + return "connector" +} diff --git a/pkg/microservice/systemconfig/core/repository/orm/connector.go b/pkg/microservice/systemconfig/core/repository/orm/connector.go new file mode 100644 index 0000000000000000000000000000000000000000..a13db56832ad960a299b8aa7de519b2811e8703f --- /dev/null +++ b/pkg/microservice/systemconfig/core/repository/orm/connector.go @@ -0,0 +1,83 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package orm + +import ( + "gorm.io/gorm" + + "github.com/koderover/zadig/pkg/microservice/systemconfig/config" + "github.com/koderover/zadig/pkg/microservice/systemconfig/core/repository/models" + gormtool "github.com/koderover/zadig/pkg/tool/gorm" +) + +type ConnectorColl struct { + *gorm.DB + + coll string +} + +func NewConnectorColl() *ConnectorColl { + name := models.Connector{}.TableName() + return &ConnectorColl{ + DB: gormtool.DB(config.MysqlDexDB()), + coll: name, + } +} + +func (c *ConnectorColl) GetCollectionName() string { + return c.coll +} + +func (c *ConnectorColl) List() ([]*models.Connector, error) { + var res []*models.Connector + result := c.Find(&res) + + return res, result.Error +} + +func (c *ConnectorColl) Get(id string) (*models.Connector, error) { + res := &models.Connector{} + result := c.First(&res, "id=?", id) + + return res, result.Error +} + +func (c *ConnectorColl) Create(obj *models.Connector) error { + obj.ResourceVersion = "1" + result := c.DB.Create(obj) + + return result.Error +} + +func (c *ConnectorColl) Update(obj *models.Connector) error { + current, err := c.Get(obj.ID) + if err != nil { + return err + } + + obj.ResourceVersion = current.ResourceVersion + obj.IncreaseResourceVersion() + result := c.DB.Save(obj) + + return result.Error +} + +func (c *ConnectorColl) Delete(id string) error { + result := c.DB.Delete(&models.Connector{ID: id}) + + return result.Error +} diff --git a/pkg/microservice/systemconfig/core/service.go b/pkg/microservice/systemconfig/core/service.go new file mode 100644 index 0000000000000000000000000000000000000000..c5c485c712d40c30c35c3b73feba8280ebd6747f --- /dev/null +++ b/pkg/microservice/systemconfig/core/service.go @@ -0,0 +1,97 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package core + +import ( + "context" + "fmt" + "sync" + "time" + + configbase "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/systemconfig/config" + "github.com/koderover/zadig/pkg/microservice/systemconfig/core/email/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/systemconfig/core/features/service" + "github.com/koderover/zadig/pkg/setting" + gormtool "github.com/koderover/zadig/pkg/tool/gorm" + "github.com/koderover/zadig/pkg/tool/log" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +func Start(_ context.Context) { + log.Init(&log.Config{ + Level: configbase.LogLevel(), + Filename: configbase.LogFile(), + SendToFile: configbase.SendLogToFile(), + Development: configbase.Mode() != setting.ReleaseMode, + }) + + initDatabase() + flagFG, err := service.FlagToFeatureGates(config.Features()) + if err != nil { + log.Errorf("FlagToFeatureGates err:%s", err) + } + dbFG, err := service.DBToFeatureGates() + if err != nil { + log.Errorf("DBToFeatureGates err:%s", err) + } + service.Features.MergeFeatureGates(flagFG, dbFG) +} + +func initDatabase() { + err := gormtool.Open(configbase.MysqlUser(), + configbase.MysqlPassword(), + configbase.MysqlHost(), + config.MysqlDexDB(), + ) + if err != nil { + log.Panicf("Failed to open database %s", config.MysqlDexDB()) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + mongotool.Init(ctx, config.MongoURI()) + if err := mongotool.Ping(ctx); err != nil { + panic(fmt.Errorf("failed to connect to mongo, error: %s", err)) + } + idxCtx, idxCancel := context.WithTimeout(ctx, 10*time.Minute) + defer idxCancel() + var wg sync.WaitGroup + for _, r := range []indexer{ + mongodb.NewEmailHostColl(), + } { + + wg.Add(1) + go func(r indexer) { + defer wg.Done() + if err := r.EnsureIndex(idxCtx); err != nil { + panic(fmt.Errorf("failed to create index for %s, error: %s", r.GetCollectionName(), err)) + } + }(r) + } + wg.Wait() +} + +type indexer interface { + EnsureIndex(ctx context.Context) error + GetCollectionName() string +} + +func Stop(_ context.Context) { + gormtool.Close() +} diff --git a/pkg/microservice/systemconfig/server/rest/router.go b/pkg/microservice/systemconfig/server/rest/router.go new file mode 100644 index 0000000000000000000000000000000000000000..affff6292b900277fbdee88c2c0e65ab5cb129b6 --- /dev/null +++ b/pkg/microservice/systemconfig/server/rest/router.go @@ -0,0 +1,43 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rest + +import ( + "github.com/gin-gonic/gin" + + codehosthandler "github.com/koderover/zadig/pkg/microservice/systemconfig/core/codehost/handler" + connectorHandler "github.com/koderover/zadig/pkg/microservice/systemconfig/core/connector/handler" + emailHandler "github.com/koderover/zadig/pkg/microservice/systemconfig/core/email/handler" + featuresHandler "github.com/koderover/zadig/pkg/microservice/systemconfig/core/features/handler" + jiraHandler "github.com/koderover/zadig/pkg/microservice/systemconfig/core/jira/handler" +) + +func (s *engine) injectRouterGroup(router *gin.RouterGroup) { + for _, r := range []injector{ + new(connectorHandler.Router), + new(emailHandler.Router), + new(jiraHandler.Router), + new(codehosthandler.Router), + new(featuresHandler.Router), + } { + r.Inject(router.Group("/api/v1")) + } +} + +type injector interface { + Inject(router *gin.RouterGroup) +} diff --git a/pkg/microservice/systemconfig/server/rest/server.go b/pkg/microservice/systemconfig/server/rest/server.go new file mode 100644 index 0000000000000000000000000000000000000000..18843b0a5ead8f194d217a281527d5152e939795 --- /dev/null +++ b/pkg/microservice/systemconfig/server/rest/server.go @@ -0,0 +1,76 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rest + +import ( + "net/http" + + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/config" + ginmiddleware "github.com/koderover/zadig/pkg/middleware/gin" + "github.com/koderover/zadig/pkg/tool/log" +) + +type engine struct { + *gin.Engine + + mode string +} + +func NewEngine() *engine { + s := &engine{mode: config.Mode()} + + gin.SetMode(s.mode) + + s.injectMiddlewares() + s.injectRouters() + + return s +} + +func (s *engine) injectMiddlewares() { + g := gin.New() + defer func() { + s.Engine = g + }() + + if s.mode == gin.TestMode { + return + } + g.Use(ginmiddleware.Response()) + g.Use(ginmiddleware.RequestID()) + g.Use(ginmiddleware.RequestLog(log.NewFileLogger(config.RequestLogFile()))) + g.Use(gin.Recovery()) +} + +func (s *engine) injectRouters() { + g := s.Engine + + g.NoRoute(func(c *gin.Context) { + c.String(http.StatusNotFound, "Invalid path: %s", c.Request.URL.Path) + }) + g.HandleMethodNotAllowed = true + g.NoMethod(func(c *gin.Context) { + c.String(http.StatusMethodNotAllowed, "Method not allowed: %s %s", c.Request.Method, c.Request.URL.Path) + }) + + apiRouters := g.Group("") + s.injectRouterGroup(apiRouters) + + s.Engine = g +} diff --git a/pkg/microservice/systemconfig/server/server.go b/pkg/microservice/systemconfig/server/server.go new file mode 100644 index 0000000000000000000000000000000000000000..cd8efc7a548dcf0ab1648b15cec61c42a89dd0cc --- /dev/null +++ b/pkg/microservice/systemconfig/server/server.go @@ -0,0 +1,60 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "context" + "net/http" + "time" + + "github.com/koderover/zadig/pkg/microservice/systemconfig/core" + "github.com/koderover/zadig/pkg/microservice/systemconfig/server/rest" + "github.com/koderover/zadig/pkg/tool/log" +) + +func Serve(ctx context.Context) error { + core.Start(ctx) + defer core.Stop(ctx) + + log.Info("Start system config service") + + engine := rest.NewEngine() + server := &http.Server{Addr: ":80", Handler: engine} + + stopChan := make(chan struct{}) + go func() { + defer close(stopChan) + + <-ctx.Done() + + ctx, cancel := context.WithTimeout(context.TODO(), 5*time.Second) + defer cancel() + + if err := server.Shutdown(ctx); err != nil { + log.Errorf("Failed to stop server, error: %s", err) + } + }() + + if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Errorf("Failed to start http server, error: %s", err) + return err + } + + <-stopChan + + return nil +} diff --git a/pkg/microservice/user/config/config.go b/pkg/microservice/user/config/config.go new file mode 100644 index 0000000000000000000000000000000000000000..3ca77a217f32256ddac6f7ce38304fe454aa6444 --- /dev/null +++ b/pkg/microservice/user/config/config.go @@ -0,0 +1,58 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "strings" + + "github.com/spf13/viper" + + _ "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/setting" +) + +func init() { + viper.SetDefault(setting.ENVUserPort, "80") +} + +func IssuerURL() string { + return viper.GetString(setting.ENVIssuerURL) +} + +func ClientID() string { + return viper.GetString(setting.ENVClientID) +} + +func ClientSecret() string { + return viper.GetString(setting.ENVClientSecret) +} + +func RedirectURI() string { + return viper.GetString(setting.ENVRedirectURI) +} + +func Scopes() []string { + return strings.Split(viper.GetString(setting.ENVScopes), ",") +} + +func MysqlUserDB() string { + return viper.GetString(setting.ENVMysqlUserDB) +} + +func TokenExpiresAt() int { + return viper.GetInt(setting.ENVTokenExpiresAt) +} diff --git a/pkg/microservice/user/config/consts.go b/pkg/microservice/user/config/consts.go new file mode 100644 index 0000000000000000000000000000000000000000..96e4e9caa60d02a82c2a28c4da4c84fbf315f917 --- /dev/null +++ b/pkg/microservice/user/config/consts.go @@ -0,0 +1,33 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "github.com/koderover/zadig/pkg/setting" +) + +const ( + AppState = setting.ProductName + "user" + SystemIdentityType = "system" + FeiShuEmailHost = "smtp.feishu.cn" +) + +type LoginType int + +const ( + AccountLoginType LoginType = 0 +) diff --git a/pkg/microservice/user/core/handler/healthz.go b/pkg/microservice/user/core/handler/healthz.go new file mode 100644 index 0000000000000000000000000000000000000000..74be72103a19978ecc845776404e071277ec1bb6 --- /dev/null +++ b/pkg/microservice/user/core/handler/healthz.go @@ -0,0 +1,30 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/user/core" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" +) + +func Healthz(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + ctx.Err = core.Healthz() +} diff --git a/pkg/microservice/user/core/handler/login/local.go b/pkg/microservice/user/core/handler/login/local.go new file mode 100644 index 0000000000000000000000000000000000000000..4df877515c21d843022a222fc91db08d568bb2ef --- /dev/null +++ b/pkg/microservice/user/core/handler/login/local.go @@ -0,0 +1,35 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package login + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/user/core/service/login" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" +) + +func LocalLogin(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + args := &login.LoginArgs{} + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return + } + ctx.Resp, ctx.Err = login.LocalLogin(args, ctx.Logger) +} diff --git a/pkg/microservice/user/core/handler/login/third_party.go b/pkg/microservice/user/core/handler/login/third_party.go new file mode 100644 index 0000000000000000000000000000000000000000..3f155b9d054e3b618218d0a84cc72446c4727cbb --- /dev/null +++ b/pkg/microservice/user/core/handler/login/third_party.go @@ -0,0 +1,175 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package login + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "github.com/coreos/go-oidc/v3/oidc" + "github.com/gin-gonic/gin" + "golang.org/x/oauth2" + + configbase "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/user/config" + "github.com/koderover/zadig/pkg/microservice/user/core/service/login" + "github.com/koderover/zadig/pkg/microservice/user/core/service/user" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/client/aslan" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/log" +) + +func provider() *oidc.Provider { + ctx := oidc.ClientContext(context.Background(), http.DefaultClient) + provider, err := oidc.NewProvider(ctx, config.IssuerURL()) + if err != nil { + log.Panicf(fmt.Sprintf("init provider error:%s", err)) + } + return provider +} + +func Login(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + oauth2Config := &oauth2.Config{ + ClientID: config.ClientID(), + ClientSecret: config.ClientSecret(), + Endpoint: provider().Endpoint(), + Scopes: config.Scopes(), + RedirectURL: config.RedirectURI(), + } + + authCodeURL := oauth2Config.AuthCodeURL(config.AppState, oauth2.AccessTypeOffline) + systemConfig, err := aslan.New(configbase.AslanServiceAddress()).GetDefaultLogin() + if err != nil { + ctx.Err = err + return + } + defaultLogin := "" + replaceURL := configbase.SystemAddress() + "/dex/auth" + if systemConfig.DefaultLogin != setting.DefaultLoginLocal { + defaultLogin = systemConfig.DefaultLogin + replaceURL = replaceURL + "/" + defaultLogin + } + authCodeURL = strings.Replace(authCodeURL, config.IssuerURL()+"/auth", replaceURL, -1) + + c.Redirect(http.StatusSeeOther, authCodeURL) +} + +func ThirdPartyLoginEnabled(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp = login.ThirdPartyLoginEnabled() +} + +func verifyAndDecode(ctx context.Context, code string) (*login.Claims, error) { + oidcCtx := oidc.ClientContext(ctx, http.DefaultClient) + oauth2Config := &oauth2.Config{ + ClientID: config.ClientID(), + ClientSecret: config.ClientSecret(), + Endpoint: provider().Endpoint(), + Scopes: nil, + RedirectURL: config.RedirectURI(), + } + var token *oauth2.Token + token, err := oauth2Config.Exchange(oidcCtx, code) + if err != nil { + return nil, e.ErrCallBackUser.AddDesc(fmt.Sprintf("failed to get token: %v", err)) + } + rawIDToken, ok := token.Extra("id_token").(string) + if !ok { + return nil, e.ErrCallBackUser.AddDesc("no id_token in token response") + } + idToken, err := provider().Verifier(&oidc.Config{ClientID: config.ClientID()}).Verify(ctx, rawIDToken) + if err != nil { + return nil, e.ErrCallBackUser.AddDesc(fmt.Sprintf("failed to verify ID token: %v", err)) + } + var claimsRaw json.RawMessage + if err := idToken.Claims(&claimsRaw); err != nil { + return nil, e.ErrCallBackUser.AddDesc(fmt.Sprintf("error decoding ID token claims: %v", err)) + } + buff := new(bytes.Buffer) + if err := json.Indent(buff, claimsRaw, "", " "); err != nil { + return nil, e.ErrCallBackUser.AddDesc(fmt.Sprintf("error indenting ID token claims: %v", err)) + } + var claims login.Claims + err = json.Unmarshal(claimsRaw, &claims) + if err != nil { + return nil, err + } + if len(claims.Name) == 0 { + claims.Name = claims.PreferredUsername + } + return &claims, nil +} + +func Callback(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + // Authorization redirect callback from OAuth2 auth flow. + if errMsg := c.Query("error"); errMsg != "" { + ctx.Err = e.ErrCallBackUser.AddDesc(errMsg) + return + } + code := c.Query("code") + if code == "" { + ctx.Err = e.ErrCallBackUser.AddDesc(fmt.Sprintf("no code in request: %q", c.Request.Form)) + return + } + if state := c.Query("state"); state != config.AppState { + ctx.Err = e.ErrCallBackUser.AddDesc(fmt.Sprintf("expected state %q got %q", config.AppState, state)) + return + } + claims, err := verifyAndDecode(c.Request.Context(), code) + if err != nil { + ctx.Err = err + return + } + + user, err := user.SyncUser(&user.SyncUserInfo{ + Account: claims.PreferredUsername, + Name: claims.Name, + Email: claims.Email, + IdentityType: claims.FederatedClaims.ConnectorId, + }, ctx.Logger) + if err != nil { + ctx.Err = err + return + } + claims.UID = user.UID + claims.StandardClaims.ExpiresAt = time.Now().Add(time.Duration(config.TokenExpiresAt()) * time.Minute).Unix() + userToken, err := login.CreateToken(claims) + if err != nil { + ctx.Err = err + return + } + v := url.Values{} + v.Add("token", userToken) + redirectUrl := "/?" + v.Encode() + c.Redirect(http.StatusSeeOther, redirectUrl) +} diff --git a/pkg/microservice/user/core/handler/router.go b/pkg/microservice/user/core/handler/router.go new file mode 100644 index 0000000000000000000000000000000000000000..f48cfbd886f7b91a661dd7d4a782badfcb6a6dd6 --- /dev/null +++ b/pkg/microservice/user/core/handler/router.go @@ -0,0 +1,67 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/user/core/handler/login" + "github.com/koderover/zadig/pkg/microservice/user/core/handler/user" +) + +type Router struct{} + +func (*Router) Inject(router *gin.RouterGroup) { + users := router.Group("") + { + users.GET("/callback", login.Callback) + + users.POST("/users", user.CreateUser) + + users.PUT("/users/:uid/password", user.UpdatePassword) + + users.PUT("/users/:uid", user.UpdateUser) + + users.PUT("/users/:uid/personal", user.UpdatePersonalUser) + + users.GET("/users/:uid", user.GetUser) + + users.DELETE("/users/:uid", user.DeleteUser) + + users.GET("/users/:uid/personal", user.GetPersonalUser) + + users.POST("/users/search", user.ListUsers) + + users.POST("/users/ldap/:ldapId", user.SyncLdapUser) + + users.GET("/user/count", user.CountSystemUsers) + + router.GET("login", login.Login) + + router.GET("login-enabled", login.ThirdPartyLoginEnabled) + + router.POST("login", login.LocalLogin) + + router.POST("signup", user.SignUp) + + router.GET("retrieve", user.Retrieve) + + router.POST("reset", user.Reset) + + router.GET("/healthz", Healthz) + } +} diff --git a/pkg/microservice/user/core/handler/user/user.go b/pkg/microservice/user/core/handler/user/user.go new file mode 100644 index 0000000000000000000000000000000000000000..d58f1f4c0b7bb939ac99f0560b6b78edd06f8a85 --- /dev/null +++ b/pkg/microservice/user/core/handler/user/user.go @@ -0,0 +1,165 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package user + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/user/config" + "github.com/koderover/zadig/pkg/microservice/user/core/service/user" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + + e "github.com/koderover/zadig/pkg/tool/errors" +) + +func SyncLdapUser(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + ldapID := c.Param("ldapId") + + ctx.Err = user.SearchAndSyncUser(ldapID, ctx.Logger) +} + +func CountSystemUsers(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = user.GetUserCount(ctx.Logger) +} + +func GetUser(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + ctx.Resp, ctx.Err = user.GetUser(c.Param("uid"), ctx.Logger) +} + +func DeleteUser(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + ctx.Err = user.DeleteUserByUID(c.Param("uid"), ctx.Logger) +} + +func GetPersonalUser(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + uid := c.Param("uid") + if ctx.UserID != uid { + ctx.Err = e.ErrForbidden + return + } + ctx.Resp, ctx.Err = user.GetUser(uid, ctx.Logger) +} + +func ListUsers(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + args := &user.QueryArgs{} + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return + } + if len(args.UIDs) > 0 { + ctx.Resp, ctx.Err = user.SearchUsersByUIDs(args.UIDs, ctx.Logger) + } else if len(args.Account) > 0 { + if len(args.IdentityType) == 0 { + args.IdentityType = config.SystemIdentityType + } + ctx.Resp, ctx.Err = user.SearchUserByAccount(args, ctx.Logger) + } else { + ctx.Resp, ctx.Err = user.SearchUsers(args, ctx.Logger) + } +} + +func CreateUser(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + args := &user.User{} + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return + } + ctx.Resp, ctx.Err = user.CreateUser(args, ctx.Logger) +} + +func UpdateUser(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + args := &user.UpdateUserInfo{} + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return + } + uid := c.Param("uid") + ctx.Err = user.UpdateUser(uid, args, ctx.Logger) +} + +func UpdatePersonalUser(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + args := &user.UpdateUserInfo{} + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return + } + uid := c.Param("uid") + if ctx.UserID != uid { + ctx.Err = e.ErrForbidden + return + } + ctx.Err = user.UpdateUser(uid, args, ctx.Logger) +} + +func SignUp(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + args := &user.User{} + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return + } + ctx.Resp, ctx.Err = user.CreateUser(args, ctx.Logger) +} + +func Retrieve(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + ctx.Resp, ctx.Err = user.Retrieve(c.Query("account"), ctx.Logger) +} + +func UpdatePassword(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + args := &user.Password{} + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return + } + args.Uid = c.Param("uid") + ctx.Err = user.UpdatePassword(args, ctx.Logger) +} + +func Reset(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + args := &user.ResetParams{} + if err := c.ShouldBindJSON(args); err != nil { + ctx.Err = err + return + } + args.Uid = ctx.UserID + ctx.Err = user.Reset(args, ctx.Logger) +} diff --git a/pkg/microservice/user/core/repository/models/base.go b/pkg/microservice/user/core/repository/models/base.go new file mode 100644 index 0000000000000000000000000000000000000000..b2682a07fbffe07f96d8fa3d927ff37a96db6a10 --- /dev/null +++ b/pkg/microservice/user/core/repository/models/base.go @@ -0,0 +1,22 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +type Model struct { + CreatedAt int64 `json:"created_at"` + UpdatedAt int64 `json:"updated_at"` +} diff --git a/pkg/microservice/user/core/repository/models/user.go b/pkg/microservice/user/core/repository/models/user.go new file mode 100644 index 0000000000000000000000000000000000000000..71fa4a342c1bbc01e4407fc054d01a944802d39d --- /dev/null +++ b/pkg/microservice/user/core/repository/models/user.go @@ -0,0 +1,33 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +type User struct { + Model + UID string `json:"uid"` + Name string `json:"name"` + IdentityType string `gorm:"default:'unknown'" json:"identity_type"` + Email string `json:"email"` + Phone string `json:"phone"` + Account string `json:"account"` + APIToken string `gorm:"api_token" json:"api_token"` +} + +// TableName sets the insert table name for this struct type +func (User) TableName() string { + return "user" +} diff --git a/pkg/microservice/user/core/repository/models/user_login.go b/pkg/microservice/user/core/repository/models/user_login.go new file mode 100644 index 0000000000000000000000000000000000000000..88a32ecf3e4cecfe41ec97b3312b519c0d4e6fda --- /dev/null +++ b/pkg/microservice/user/core/repository/models/user_login.go @@ -0,0 +1,31 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +type UserLogin struct { + Model + UID string `json:"uid"` + Password string `json:"password"` + LastLoginTime int64 `json:"last_login_time"` + LoginId string `json:"login_id"` + LoginType int `json:"login_type"` +} + +// TableName sets the insert table name for this struct type +func (UserLogin) TableName() string { + return "user_login" +} diff --git a/pkg/microservice/user/core/repository/orm/user.go b/pkg/microservice/user/core/repository/orm/user.go new file mode 100644 index 0000000000000000000000000000000000000000..56ee417ba865545688f21b34ddb294a3a7f53238 --- /dev/null +++ b/pkg/microservice/user/core/repository/orm/user.go @@ -0,0 +1,161 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package orm + +import ( + "gorm.io/gorm" + + "github.com/koderover/zadig/pkg/microservice/user/core" + "github.com/koderover/zadig/pkg/microservice/user/core/repository/models" + "github.com/koderover/zadig/pkg/types" +) + +// CreateUser create a user +func CreateUser(user *models.User, db *gorm.DB) error { + if err := db.Create(&user).Error; err != nil { + return err + } + return nil +} + +// GetUser Get a user based on email and identityType +func GetUser(account string, identityType string, db *gorm.DB) (*models.User, error) { + var user models.User + err := db.Where("account = ? and identity_type = ?", account, identityType).First(&user).Error + if err != nil && err != gorm.ErrRecordNotFound { + return nil, err + } + if err == gorm.ErrRecordNotFound { + return nil, nil + } + return &user, nil +} + +// GetUserByUid Get a user based on uid +func GetUserByUid(uid string, db *gorm.DB) (*models.User, error) { + var user models.User + err := db.Where("uid = ?", uid).First(&user).Error + if err != nil && err != gorm.ErrRecordNotFound { + return nil, err + } + if err == gorm.ErrRecordNotFound { + return nil, nil + } + return &user, nil +} + +// ListUsers gets a list of users based on paging constraints +func ListUsers(page int, perPage int, name string, db *gorm.DB) ([]models.User, error) { + var ( + users []models.User + err error + ) + + err = db.Where("name LIKE ?", "%"+name+"%").Order("account ASC").Offset((page - 1) * perPage).Limit(perPage).Find(&users).Error + + if err != nil && err != gorm.ErrRecordNotFound { + return nil, err + } + + return users, nil +} + +// ListUsersByUIDs gets a list of users based on paging constraints +func ListUsersByUIDs(uids []string, db *gorm.DB) ([]models.User, error) { + var ( + users []models.User + err error + ) + + err = db.Find(&users, "uid in ?", uids).Error + + if err != nil && err != gorm.ErrRecordNotFound { + return nil, err + } + + return users, nil +} + +// ListUsersByIdentityType gets a list of users based on identityType +func ListUsersByIdentityType(identityType string, db *gorm.DB) ([]models.User, error) { + var ( + users []models.User + err error + ) + + err = db.Find(&users, "identity_type = ?", identityType).Error + + if err != nil && err != gorm.ErrRecordNotFound { + return nil, err + } + + return users, nil +} + +// DeleteUserByUids Delete users based on uids +func DeleteUserByUids(uids []string, db *gorm.DB) error { + var user models.User + err := db.Where("uid in ?", uids).Delete(&user).Error + if err != nil { + return err + } + return nil +} + +// DeleteUserByUid Delete users based on uids +func DeleteUserByUid(uid string, db *gorm.DB) error { + var user models.User + err := db.Where("uid = ?", uid).Delete(&user).Error + if err != nil { + return err + } + return nil +} + +// GetUsersCount gets user count +func GetUsersCount(name string) (int64, error) { + var ( + users []models.User + err error + count int64 + ) + + err = core.DB.Where("name LIKE ?", "%"+name+"%").Find(&users).Count(&count).Error + + if err != nil { + return 0, err + } + + return count, nil +} + +// UpdateUser update user info +func UpdateUser(uid string, user *models.User, db *gorm.DB) error { + if err := db.Model(&models.User{}).Where("uid = ?", uid).Updates(user).Error; err != nil { + return err + } + return nil +} + +func CountUserByType(db *gorm.DB) ([]*types.UserCountByType, error) { + var resp []*types.UserCountByType + err := db.Model(&models.User{}).Select("count(*) as count, identity_type").Group("identity_type").Find(&resp).Error + if err != nil { + return nil, err + } + return resp, nil +} diff --git a/pkg/microservice/user/core/repository/orm/user_login.go b/pkg/microservice/user/core/repository/orm/user_login.go new file mode 100644 index 0000000000000000000000000000000000000000..361c485f4301fc940e08ace62d0400d512b7c14e --- /dev/null +++ b/pkg/microservice/user/core/repository/orm/user_login.go @@ -0,0 +1,92 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package orm + +import ( + "gorm.io/gorm" + + "github.com/koderover/zadig/pkg/microservice/user/config" + "github.com/koderover/zadig/pkg/microservice/user/core/repository/models" +) + +// CreateUserLogin add a userLogin record +func CreateUserLogin(userLogin *models.UserLogin, db *gorm.DB) error { + if err := db.Create(&userLogin).Error; err != nil { + return err + } + return nil +} + +// GetUserLogin Get a userLogin based on uid +func GetUserLogin(uid string, account string, loginType config.LoginType, db *gorm.DB) (*models.UserLogin, error) { + var userLogin models.UserLogin + err := db.Where("uid = ? and login_id = ? and login_type = ?", uid, account, loginType).First(&userLogin).Error + if err != nil && err != gorm.ErrRecordNotFound { + return nil, err + } + if err == gorm.ErrRecordNotFound { + return nil, nil + } + return &userLogin, nil +} + +// DeleteUserLoginByUids Delete userLogin based on uids +func DeleteUserLoginByUids(uids []string, db *gorm.DB) error { + var userLogin models.UserLogin + err := db.Where("uid in ?", uids).Delete(&userLogin).Error + if err != nil { + return err + } + return nil +} + +// DeleteUserLoginByUid Delete userLogin based on uids +func DeleteUserLoginByUid(uid string, db *gorm.DB) error { + var userLogin models.UserLogin + err := db.Where("uid = ?", uid).Delete(&userLogin).Error + if err != nil { + return err + } + return nil +} + +// ListUserLogins Get a userLogin based on uid list +func ListUserLogins(uids []string, db *gorm.DB) (*[]models.UserLogin, error) { + var userLogins []models.UserLogin + err := db.Find(&userLogins, "uid in ?", uids).Error + if err != nil { + return nil, err + } + return &userLogins, nil +} + +// UpdateUserLogin update login info +func UpdateUserLogin(uid string, userLogin *models.UserLogin, db *gorm.DB) error { + if err := db.Model(&models.UserLogin{}).Where("uid = ?", uid).Updates(userLogin).Error; err != nil { + return err + } + return nil +} + +func CountUser(db *gorm.DB) (int64, error) { + var count int64 + err := db.Model(&models.UserLogin{}).Select("count(*)").Where("last_login_time > 0").Find(&count).Error + if err != nil { + return 0, err + } + return count, nil +} diff --git a/pkg/microservice/user/core/service.go b/pkg/microservice/user/core/service.go new file mode 100644 index 0000000000000000000000000000000000000000..89296880488d910ccf32bdc450899c5dfd1e2f5c --- /dev/null +++ b/pkg/microservice/user/core/service.go @@ -0,0 +1,95 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package core + +import ( + "context" + "database/sql" + _ "embed" + "fmt" + + _ "github.com/go-sql-driver/mysql" + "gorm.io/gorm" + + "github.com/koderover/zadig/pkg/config" + config2 "github.com/koderover/zadig/pkg/microservice/user/config" + "github.com/koderover/zadig/pkg/setting" + gormtool "github.com/koderover/zadig/pkg/tool/gorm" + "github.com/koderover/zadig/pkg/tool/log" +) + +var DB *gorm.DB + +//go:embed service/init/mysql.sql +var mysql []byte + +func initDBAndTables() { + if len(mysql) == 0 { + return + } + db, err := sql.Open("mysql", fmt.Sprintf( + "%s:%s@tcp(%s)/?charset=utf8&multiStatements=true", + config.MysqlUser(), config.MysqlPassword(), config.MysqlHost(), + )) + if err != nil { + log.Panic(err) + } + defer db.Close() + initSql := fmt.Sprintf(string(mysql), config.MysqlUserDB(), config.MysqlUserDB()) + _, err = db.Exec(initSql) + + if err != nil { + log.Panic(err) + } +} + +func Start(_ context.Context) { + log.Init(&log.Config{ + Level: config.LogLevel(), + Filename: config.LogFile(), + SendToFile: config.SendLogToFile(), + Development: config.Mode() != setting.ReleaseMode, + }) + + initDatabase() + DB = gormtool.DB(config2.MysqlUserDB()) +} + +func initDatabase() { + initDBAndTables() + err := gormtool.Open(config.MysqlUser(), + config.MysqlPassword(), + config.MysqlHost(), + config2.MysqlUserDB(), + ) + if err != nil { + log.Panicf("Failed to open database %s", config2.MysqlUserDB()) + } +} + +func Stop(_ context.Context) { + gormtool.Close() +} + +func Healthz() error { + sqlDB, err := DB.DB() + if err != nil { + log.Errorf("Healthz get db error:%s", err.Error()) + return err + } + return sqlDB.Ping() +} diff --git a/pkg/microservice/user/core/service/init/mysql.sql b/pkg/microservice/user/core/service/init/mysql.sql new file mode 100644 index 0000000000000000000000000000000000000000..a2fcd0a8ca2fc62d244450e0ebc23ba24f3f8850 --- /dev/null +++ b/pkg/microservice/user/core/service/init/mysql.sql @@ -0,0 +1,29 @@ +CREATE DATABASE IF NOT EXISTS `%s`; +use `%s`; +CREATE TABLE IF NOT EXISTS `user_login`( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `uid` varchar(64) NOT NULL DEFAULT '0' COMMENT '用户id', + `login_id` varchar(64) NOT NULL DEFAULT '0' COMMENT '用户登录id,如账号名', + `login_type` int(4) unsigned NOT NULL DEFAULT '0' COMMENT '登录类型,0.账号名', + `password` varchar(64) DEFAULT '' COMMENT '密码', + `last_login_time` int(11) unsigned NOT NULL DEFAULT '0' COMMENT '最后登陆时间', + `created_at` int(11) unsigned NOT NULL DEFAULT '0' COMMENT '创建时间', + `updated_at` int(11) unsigned NOT NULL DEFAULT '0' COMMENT '更新时间', + UNIQUE KEY `login` (`uid`,`login_id`,`login_type`), + PRIMARY KEY (`id`), + KEY `idx_uid` (`uid`) USING BTREE +) ENGINE = InnoDB AUTO_INCREMENT = 59 CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '账号登陆表' ROW_FORMAT = Compact; + +CREATE TABLE IF NOT EXISTS `user`( + `uid` varchar(64) NOT NULL COMMENT '用户ID', + `account` varchar(32) NOT NULL DEFAULT '' COMMENT '用户账号', + `name` varchar(32) NOT NULL DEFAULT '' COMMENT '用户名', + `identity_type` varchar(32) NOT NULL DEFAULT 'unknown' COMMENT '用户来源', + `phone` varchar(16) NOT NULL DEFAULT '' COMMENT '手机号码', + `email` varchar(100) NOT NULL DEFAULT '' COMMENT '邮箱', + `api_token` varchar(1024) NOT NULL DEFAULT '' COMMENT 'openAPIToken', + `created_at` int(11) unsigned NOT NULL COMMENT '创建时间', + `updated_at` int(11) unsigned NOT NULL COMMENT '修改时间', + UNIQUE KEY `account` (`account`,`identity_type`), + PRIMARY KEY (`uid`) +) ENGINE = InnoDB AUTO_INCREMENT = 59 CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '用户信息表' ROW_FORMAT = Compact; \ No newline at end of file diff --git a/pkg/microservice/user/core/service/login/local.go b/pkg/microservice/user/core/service/login/local.go new file mode 100644 index 0000000000000000000000000000000000000000..eb1254142403e2aa9645091c40c0bd5572aa2bcb --- /dev/null +++ b/pkg/microservice/user/core/service/login/local.go @@ -0,0 +1,146 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package login + +import ( + "fmt" + "time" + + "github.com/golang-jwt/jwt" + "go.uber.org/zap" + "golang.org/x/crypto/bcrypt" + + "github.com/koderover/zadig/pkg/microservice/user/config" + "github.com/koderover/zadig/pkg/microservice/user/core" + "github.com/koderover/zadig/pkg/microservice/user/core/repository/orm" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/client/plutusvendor" +) + +type LoginArgs struct { + Account string `json:"account"` + Password string `json:"password"` +} + +type User struct { + Uid string `json:"uid"` + Token string `json:"token"` + Email string `json:"email"` + Phone string `json:"phone"` + Name string `json:"name"` + Account string `json:"account"` + IdentityType string `json:"identityType"` +} + +type CheckSignatureRes struct { + Code int `json:"code"` + Description string `json:"description"` + Extra map[string]interface{} `json:"extra"` + Message string `json:"message"` + Type string `json:"type"` +} + +func CheckSignature(ifLoggedIn bool, logger *zap.SugaredLogger) error { + userNum, err := orm.CountUser(core.DB) + if err != nil { + return err + } + vendorClient := plutusvendor.New() + err = vendorClient.Health() + if err == nil { + res, checkErr := vendorClient.CheckSignature(userNum) + if checkErr != nil { + return checkErr + } + if res.Code == 6694 { + if ifLoggedIn { + return nil + } else { + return fmt.Errorf("系统使用用户数量已达授权人数上限，请联系系统管理员") + } + + } + } + return nil +} + +func LocalLogin(args *LoginArgs, logger *zap.SugaredLogger) (*User, error) { + user, err := orm.GetUser(args.Account, config.SystemIdentityType, core.DB) + if err != nil { + logger.Errorf("InternalLogin get user account:%s error", args.Account) + return nil, err + } + if user == nil { + return nil, fmt.Errorf("user not exist") + } + userLogin, err := orm.GetUserLogin(user.UID, args.Account, config.AccountLoginType, core.DB) + if err != nil { + logger.Errorf("LocalLogin get user:%s user login not exist, error msg:%s", args.Account, err.Error()) + return nil, err + } + if userLogin == nil { + logger.Errorf("InternalLogin user:%s user login not exist", args.Account) + return nil, fmt.Errorf("user login not exist") + } + password := []byte(args.Password) + err = bcrypt.CompareHashAndPassword([]byte(userLogin.Password), password) + if err == bcrypt.ErrMismatchedHashAndPassword { + return nil, fmt.Errorf("password is wrong") + } + if err != nil { + logger.Errorf("LocalLogin user:%s check password error, error msg:%s", args.Account, err) + return nil, fmt.Errorf("check password error, error msg:%s", err) + } + err = CheckSignature(userLogin.LastLoginTime > 0, logger) + if err != nil { + return nil, err + } + userLogin.LastLoginTime = time.Now().Unix() + err = orm.UpdateUserLogin(userLogin.UID, userLogin, core.DB) + if err != nil { + logger.Errorf("LocalLogin user:%s update user login password error, error msg:%s", args.Account, err.Error()) + return nil, err + } + token, err := CreateToken(&Claims{ + Name: user.Name, + UID: user.UID, + Email: user.Email, + PreferredUsername: user.Account, + StandardClaims: jwt.StandardClaims{ + Audience: setting.ProductName, + ExpiresAt: time.Now().Add(24 * time.Hour).Unix(), + }, + FederatedClaims: FederatedClaims{ + ConnectorId: user.IdentityType, + UserId: user.Account, + }, + }) + if err != nil { + logger.Errorf("LocalLogin user:%s create token error, error msg:%s", args.Account, err.Error()) + return nil, err + } + + return &User{ + Uid: user.UID, + Token: token, + Email: user.Email, + Phone: user.Phone, + Name: user.Name, + Account: user.Account, + IdentityType: user.IdentityType, + }, nil +} diff --git a/pkg/microservice/user/core/service/login/third_party.go b/pkg/microservice/user/core/service/login/third_party.go new file mode 100644 index 0000000000000000000000000000000000000000..5d0919b402010de782f71d5caef85d0dd3258d77 --- /dev/null +++ b/pkg/microservice/user/core/service/login/third_party.go @@ -0,0 +1,36 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package login + +import ( + "github.com/koderover/zadig/pkg/shared/client/systemconfig" + "github.com/koderover/zadig/pkg/tool/log" +) + +type enabledStatus struct { + Enabled bool `json:"enabled"` +} + +func ThirdPartyLoginEnabled() *enabledStatus { + connectors, err := systemconfig.New().ListConnectorsInternal() + if err != nil { + log.Warnf("Failed to list connectors, err: %s", err) + } + return &enabledStatus{ + Enabled: len(connectors) > 0, + } +} diff --git a/pkg/microservice/user/core/service/login/token.go b/pkg/microservice/user/core/service/login/token.go new file mode 100644 index 0000000000000000000000000000000000000000..d2e59c21159ff787eefa522f96ab347fe3c2c161 --- /dev/null +++ b/pkg/microservice/user/core/service/login/token.go @@ -0,0 +1,46 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package login + +import ( + "github.com/golang-jwt/jwt" + + "github.com/koderover/zadig/pkg/config" +) + +type Claims struct { + Name string `json:"name"` + Email string `json:"email"` + UID string `json:"uid"` + PreferredUsername string `json:"preferred_username"` + FederatedClaims FederatedClaims `json:"federated_claims"` + jwt.StandardClaims +} + +type FederatedClaims struct { + ConnectorId string `json:"connector_id"` + UserId string `json:"user_id"` +} + +func CreateToken(claims *Claims) (string, error) { + token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + tokenString, err := token.SignedString([]byte(config.SecretKey())) + if err != nil { + return "", err + } + return tokenString, nil +} diff --git a/pkg/microservice/user/core/service/user/retrieve.html b/pkg/microservice/user/core/service/user/retrieve.html new file mode 100644 index 0000000000000000000000000000000000000000..87f024a4b8214ed32c1e94ae6e56204139255be4 --- /dev/null +++ b/pkg/microservice/user/core/service/user/retrieve.html @@ -0,0 +1,34 @@ +
+
+ + + + + + +
hi，您好
+
+
+ +
+
+

非常感谢使用 Zadig，点击下面的链接完成重置密码。

+
+
+ + + + + + + + + +
{{.Url}}
如果您不想继续完成密码重置，请忽略该邮件，链接将在5分钟之后过期!
+
+
+ +
+

本邮件由 Zadig 系统自动发出，请勿直接回复。

+

Made By Zadig Team ♥ Happy Coding.

+
\ No newline at end of file diff --git a/pkg/microservice/user/core/service/user/user.go b/pkg/microservice/user/core/service/user/user.go new file mode 100644 index 0000000000000000000000000000000000000000..0a6be6d67fdc325819ca769c2903c40c9ec454e5 --- /dev/null +++ b/pkg/microservice/user/core/service/user/user.go @@ -0,0 +1,604 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package user + +import ( + _ "embed" + "errors" + "fmt" + "net/url" + "time" + + "github.com/dexidp/dex/connector/ldap" + ldapv3 "github.com/go-ldap/ldap/v3" + "github.com/go-sql-driver/mysql" + "github.com/golang-jwt/jwt" + "github.com/google/uuid" + "go.uber.org/zap" + "golang.org/x/crypto/bcrypt" + + configbase "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/user/config" + "github.com/koderover/zadig/pkg/microservice/user/core" + "github.com/koderover/zadig/pkg/microservice/user/core/repository/models" + "github.com/koderover/zadig/pkg/microservice/user/core/repository/orm" + "github.com/koderover/zadig/pkg/microservice/user/core/service/login" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/mail" + "github.com/koderover/zadig/pkg/types" +) + +type User struct { + Name string `json:"name"` + Password string `json:"password"` + Email string `json:"email"` + Account string `json:"account"` + Phone string `json:"phone,omitempty"` +} + +type UpdateUserInfo struct { + Name string `json:"name,omitempty"` + Email string `json:"email,omitempty"` + Phone string `json:"phone,omitempty"` +} + +type QueryArgs struct { + Name string `json:"name,omitempty"` + Account string `json:"account,omitempty"` + IdentityType string `json:"identity_type,omitempty"` + UIDs []string `json:"uids,omitempty"` + PerPage int `json:"per_page,omitempty"` + Page int `json:"page,omitempty"` +} + +type Password struct { + Uid string `json:"uid"` + OldPassword string `json:"oldPassword"` + NewPassword string `json:"newPassword"` +} + +type ResetParams struct { + Uid string `json:"uid"` + Password string `json:"password"` +} + +type SyncUserInfo struct { + Account string `json:"account"` + IdentityType string `json:"identityType"` + Name string `json:"name"` + Email string `json:"email"` +} + +type RetrieveResp struct { + Email string `json:"email"` +} + +func SearchAndSyncUser(ldapId string, logger *zap.SugaredLogger) error { + systemConfigClient := systemconfig.New() + si, err := systemConfigClient.GetLDAPConnector(ldapId) + if err != nil { + logger.Errorf("SearchAndSyncUser GetLDAPConnector error, error msg:%s", err) + return fmt.Errorf("SearchAndSyncUser GetLDAPConnector error, error msg:%s", err) + } + if si == nil || si.Config == nil { + logger.Error("can't find connector") + return fmt.Errorf("can't find connector") + } + + config := si.Config.(*ldap.Config) + l, err := ldapv3.Dial("tcp", config.Host) + if err != nil { + logger.Errorf("ldap dial host:%s error, error msg:%s", config.Host, err) + return err + } + defer l.Close() + + err = l.Bind(config.BindDN, config.BindPW) + if err != nil { + logger.Errorf("ldap bind host:%s error, error msg:%s", config.Host, err) + return err + } + + searchRequest := ldapv3.NewSearchRequest( + config.GroupSearch.BaseDN, + ldapv3.ScopeWholeSubtree, ldapv3.NeverDerefAliases, 0, 0, false, + config.GroupSearch.Filter, // The filter to apply + []string{config.GroupSearch.NameAttr, config.UserSearch.NameAttr, config.UserSearch.PreferredUsernameAttrAttr, + config.UserSearch.EmailAttr}, // A list attributes to retrieve + nil, + ) + + sr, err := l.Search(searchRequest) + if err != nil { + logger.Errorf("ldap search host:%s error, error msg:%s", config.Host, err) + return err + } + for _, entry := range sr.Entries { + account := config.UserSearch.PreferredUsernameAttrAttr + name := account + if len(config.UserSearch.NameAttr) != 0 { + name = config.UserSearch.NameAttr + } + _, err := SyncUser(&SyncUserInfo{ + Account: entry.GetAttributeValue(account), + Name: entry.GetAttributeValue(name), + Email: entry.GetAttributeValue(config.UserSearch.EmailAttr), + IdentityType: si.ID, // ldap may have not only one instance, so use id as identityType + }, logger) + if err != nil { + logger.Errorf("ldap host:%s sync user error, error msg:%s", config.Host, err) + return err + } + } + return nil +} + +func GetUser(uid string, logger *zap.SugaredLogger) (*types.UserInfo, error) { + user, err := orm.GetUserByUid(uid, core.DB) + if err != nil { + logger.Errorf("GetUser getUserByUid:%s error, error msg:%s", uid, err.Error()) + return nil, err + } + if user == nil { + return nil, nil + } + userLogin, err := orm.GetUserLogin(uid, user.Account, config.AccountLoginType, core.DB) + if err != nil { + logger.Errorf("GetUser GetUserLogin:%s error, error msg:%s", uid, err.Error()) + return nil, err + } + userInfo := mergeUserLogin([]models.User{*user}, []models.UserLogin{*userLogin}, logger) + userInfoRes := &userInfo[0] + userInfoRes.APIToken = user.APIToken + //TODO Create a permanent OpenAPI token + if user.APIToken == "" { + token, err := login.CreateToken(&login.Claims{ + Name: user.Name, + UID: user.UID, + Email: user.Email, + PreferredUsername: user.Account, + StandardClaims: jwt.StandardClaims{ + Audience: setting.ProductName, + //24*365*100=876000 + ExpiresAt: time.Now().Add(876000 * time.Hour).Unix(), + }, + FederatedClaims: login.FederatedClaims{ + ConnectorId: user.IdentityType, + UserId: user.Account, + }, + }) + if err != nil { + logger.Errorf("LocalLogin user:%s create token error, error msg:%s", user.Account, err.Error()) + return nil, err + } + userInfoRes.APIToken = token + userWithToken := &models.User{ + APIToken: token, + } + err = orm.UpdateUser(uid, userWithToken, core.DB) + if err != nil { + logger.Errorf("UpdateUser user:%s save token error:%s", user.Account, err.Error()) + return nil, err + } + } + return userInfoRes, nil +} + +func SearchUserByAccount(args *QueryArgs, logger *zap.SugaredLogger) (*types.UsersResp, error) { + user, err := orm.GetUser(args.Account, args.IdentityType, core.DB) + if err != nil { + logger.Errorf("SearchUserByAccount GetUser By account:%s error, error msg:%s", args.Account, err.Error()) + return nil, err + } + if user == nil { + return &types.UsersResp{ + Users: nil, + TotalCount: 0, + }, nil + } + userLogins, err := orm.ListUserLogins([]string{user.UID}, core.DB) + if err != nil { + logger.Errorf("SearchUserByAccount ListUserLogins By uid:%s error, error msg:%s", user.UID, err.Error()) + return nil, err + } + usersInfo := mergeUserLogin([]models.User{*user}, *userLogins, logger) + return &types.UsersResp{ + Users: usersInfo, + TotalCount: int64(len(usersInfo)), + }, nil +} + +func SearchUsers(args *QueryArgs, logger *zap.SugaredLogger) (*types.UsersResp, error) { + count, err := orm.GetUsersCount(args.Name) + if err != nil { + logger.Errorf("SeachUsers GetUsersCount By name:%s error, error msg:%s", args.Name, err.Error()) + return nil, err + } + if count == 0 { + return &types.UsersResp{ + TotalCount: 0, + }, nil + } + + users, err := orm.ListUsers(args.Page, args.PerPage, args.Name, core.DB) + if err != nil { + logger.Errorf("SeachUsers SeachUsers By name:%s error, error msg:%s", args.Name, err.Error()) + return nil, err + } + var uids []string + for _, user := range users { + uids = append(uids, user.UID) + } + userLogins, err := orm.ListUserLogins(uids, core.DB) + if err != nil { + logger.Errorf("SeachUsers ListUserLogins By uids:%s error, error msg:%s", uids, err.Error()) + return nil, err + } + usersInfo := mergeUserLogin(users, *userLogins, logger) + return &types.UsersResp{ + Users: usersInfo, + TotalCount: count, + }, nil +} + +func mergeUserLogin(users []models.User, userLogins []models.UserLogin, logger *zap.SugaredLogger) []types.UserInfo { + userLoginMap := make(map[string]models.UserLogin) + for _, userLogin := range userLogins { + userLoginMap[userLogin.UID] = userLogin + } + var usersInfo []types.UserInfo + for _, user := range users { + if userLogin, ok := userLoginMap[user.UID]; ok { + usersInfo = append(usersInfo, types.UserInfo{ + LastLoginTime: userLogin.LastLoginTime, + Uid: user.UID, + Phone: user.Phone, + Name: user.Name, + Email: user.Email, + IdentityType: user.IdentityType, + Account: user.Account, + }) + } else { + logger.Error("user:%s login info not exist") + } + } + return usersInfo +} + +func SearchUsersByUIDs(uids []string, logger *zap.SugaredLogger) (*types.UsersResp, error) { + users, err := orm.ListUsersByUIDs(uids, core.DB) + if err != nil { + logger.Errorf("SearchUsersByUIDs SeachUsers By uids:%s error, error msg:%s", uids, err.Error()) + return nil, err + } + userLogins, err := orm.ListUserLogins(uids, core.DB) + if err != nil { + logger.Errorf("SearchUsersByUIDs ListUserLogins By uids:%s error, error msg:%s", uids, err.Error()) + return nil, err + } + usersInfo := mergeUserLogin(users, *userLogins, logger) + return &types.UsersResp{ + Users: usersInfo, + TotalCount: int64(len(usersInfo)), + }, nil +} + +func getLoginId(user *models.User, loginType config.LoginType) string { + switch loginType { + case config.AccountLoginType: + return user.Account + default: + return user.Account + } + +} + +func DeleteUserByUID(uid string, logger *zap.SugaredLogger) error { + tx := core.DB.Begin() + defer func() { + if r := recover(); r != nil { + tx.Rollback() + } + }() + err := orm.DeleteUserByUid(uid, tx) + if err != nil { + tx.Rollback() + logger.Errorf("DeleteUserByUID DeleteUserByUid :%s error, error msg:%s", uid, err.Error()) + return err + } + err = orm.DeleteUserLoginByUid(uid, tx) + if err != nil { + tx.Rollback() + logger.Errorf("DeleteUserByUID DeleteUserLoginByUid:%s error, error msg:%s", uid, err.Error()) + return err + } + return tx.Commit().Error +} + +//go:embed retrieve.html +var retrieveHemlTemplate []byte + +func Retrieve(account string, logger *zap.SugaredLogger) (*RetrieveResp, error) { + user, err := orm.GetUser(account, config.SystemIdentityType, core.DB) + if err != nil { + logger.Errorf("Retrieve GetUser:%s error, error msg:%s ", account, err) + return nil, fmt.Errorf("Retrieve GetUser:%s error, error msg:%s ", account, err) + } + if user == nil { + return nil, fmt.Errorf("user not exist") + } + if len(user.Email) == 0 { + logger.Errorf("the account:%s has not email", account) + return nil, fmt.Errorf("the account has not email") + } + + token, err := login.CreateToken(&login.Claims{ + Name: user.Name, + UID: user.UID, + Email: user.Email, + StandardClaims: jwt.StandardClaims{ + Audience: setting.ProductName, + ExpiresAt: time.Now().Add(5 * time.Minute).Unix(), + }, + FederatedClaims: login.FederatedClaims{ + UserId: user.Account, + ConnectorId: user.IdentityType, + }, + }) + if err != nil { + logger.Errorf("Retrieve user:%s create token error, error msg:%s", user.Account, err) + return nil, err + } + v := url.Values{} + v.Add("idtoken", token) + retrieveURL := configbase.SystemAddress() + "/signin?" + v.Encode() + body, err := mail.RenderEmailTemplate(retrieveURL, string(retrieveHemlTemplate)) + if err != nil { + logger.Errorf("Retrieve renderEmailTemplate error, error msg:%s ", err) + return nil, fmt.Errorf("Retrieve renderEmailTemplate error, error msg:%s ", err) + } + systemConfigClient := systemconfig.New() + email, err := systemConfigClient.GetEmailHost() + if err != nil { + logger.Errorf("Retrieve GetEmailHost error, error msg:%s", err) + return nil, fmt.Errorf("Retrieve GetEmailHost error, error msg:%s ", err) + } + err = mail.SendEmail(&mail.EmailParams{ + From: email.UserName, + To: user.Email, + Subject: "重置密码", + Host: email.Name, + UserName: email.UserName, + Password: email.Password, + Port: email.Port, + Body: body, + }) + if err != nil { + logger.Errorf("Retrieve SendEmail error, error msg:%s ", err) + return nil, err + } + return &RetrieveResp{ + Email: user.Email, + }, nil +} + +func CreateUser(args *User, logger *zap.SugaredLogger) (*models.User, error) { + uid, _ := uuid.NewUUID() + user := &models.User{ + Name: args.Name, + Email: args.Email, + IdentityType: config.SystemIdentityType, + Phone: args.Phone, + Account: args.Account, + UID: uid.String(), + } + tx := core.DB.Begin() + defer func() { + if r := recover(); r != nil { + tx.Rollback() + } + }() + err := orm.CreateUser(user, tx) + if err != nil { + tx.Rollback() + logger.Errorf("CreateUser CreateUser :%v error, error msg:%s", user, err.Error()) + var mysqlErr *mysql.MySQLError + if errors.As(err, &mysqlErr) && mysqlErr.Number == 1062 { + return nil, e.ErrCreateUser.AddErr(err).AddDesc("存在相同用户名") + } + return nil, e.ErrCreateUser.AddErr(err) + } + hashedPassword, _ := bcrypt.GenerateFromPassword([]byte(args.Password), bcrypt.DefaultCost) + userLogin := &models.UserLogin{ + UID: user.UID, + Password: string(hashedPassword), + LastLoginTime: 0, + LoginId: getLoginId(user, config.AccountLoginType), + LoginType: int(config.AccountLoginType), + } + err = orm.CreateUserLogin(userLogin, tx) + if err != nil { + tx.Rollback() + logger.Errorf("CreateUser CreateUserLogin:%v error, error msg:%s", user, err.Error()) + return nil, err + } + return user, tx.Commit().Error +} + +func UpdateUser(uid string, args *UpdateUserInfo, _ *zap.SugaredLogger) error { + user := &models.User{ + Name: args.Name, + Email: args.Email, + Phone: args.Phone, + } + return orm.UpdateUser(uid, user, core.DB) + +} + +func UpdatePassword(args *Password, logger *zap.SugaredLogger) error { + user, err := orm.GetUserByUid(args.Uid, core.DB) + if err != nil { + logger.Errorf("UpdatePassword GetUserByUid:%s error, error msg:%s", args.Uid, err.Error()) + return err + } + if user == nil { + return fmt.Errorf("user not exist") + } + userLogin, err := orm.GetUserLogin(user.UID, user.Account, config.AccountLoginType, core.DB) + if err != nil { + logger.Errorf("UpdatePassword GetUserLogin:%s error, error msg:%s", args.Uid, err.Error()) + return err + } + if userLogin == nil { + logger.Errorf("UpdatePassword GetUserLogin:%s not exist", args.Uid) + return fmt.Errorf("userLogin not exist") + } + password := []byte(args.OldPassword) + err = bcrypt.CompareHashAndPassword([]byte(userLogin.Password), password) + if err == bcrypt.ErrMismatchedHashAndPassword { + return fmt.Errorf("password is wrong") + } + if err != nil { + logger.Errorf("UpdatePassword CompareHashAndPassword userLogin password:%s, password:%s error,"+ + " error msg:%s", userLogin.Password, password, err.Error()) + return err + } + hashedPassword, _ := bcrypt.GenerateFromPassword([]byte(args.NewPassword), bcrypt.DefaultCost) + userLogin = &models.UserLogin{ + UID: user.UID, + Password: string(hashedPassword), + } + err = orm.UpdateUserLogin(user.UID, userLogin, core.DB) + if err != nil { + logger.Errorf("UpdatePassword UpdateUserLogin:%v error, error msg:%s", userLogin, err.Error()) + return err + } + return nil +} + +func Reset(args *ResetParams, logger *zap.SugaredLogger) error { + user, err := orm.GetUserByUid(args.Uid, core.DB) + if err != nil { + logger.Errorf("Reset GetUserByUid:%s error, error msg:%s", args.Uid, err) + return err + } + if user == nil { + logger.Error("user not exist") + return fmt.Errorf("user not exist") + } + + hashedPassword, _ := bcrypt.GenerateFromPassword([]byte(args.Password), bcrypt.DefaultCost) + userLogin := &models.UserLogin{ + UID: user.UID, + Password: string(hashedPassword), + } + err = orm.UpdateUserLogin(user.UID, userLogin, core.DB) + if err != nil { + logger.Errorf("UpdatePassword UpdateUserLogin:%v error, error msg:%s", userLogin, err.Error()) + return err + } + return nil +} + +func SyncUser(syncUserInfo *SyncUserInfo, logger *zap.SugaredLogger) (*models.User, error) { + user, err := orm.GetUser(syncUserInfo.Account, syncUserInfo.IdentityType, core.DB) + if err != nil { + logger.Error("SyncUser get user:%s error, error msg:%s", syncUserInfo.Account, err.Error()) + return nil, err + } + tx := core.DB.Begin() + defer func() { + if r := recover(); r != nil { + tx.Rollback() + } + }() + if user == nil { + uid, _ := uuid.NewUUID() + user = &models.User{ + UID: uid.String(), + Name: syncUserInfo.Name, + Account: syncUserInfo.Account, + Email: syncUserInfo.Email, + IdentityType: syncUserInfo.IdentityType, + } + err = orm.CreateUser(user, tx) + if err != nil { + tx.Rollback() + logger.Error("SyncUser create user:%s error, error msg:%s", syncUserInfo.Account, err.Error()) + return nil, err + } + } else { + err = orm.UpdateUser(user.UID, &models.User{ + Name: syncUserInfo.Name, + Account: syncUserInfo.Account, + Email: syncUserInfo.Email, + }, tx) + if err != nil { + tx.Rollback() + logger.Error("SyncUser update user:%s error, error msg:%s", syncUserInfo.Account, err.Error()) + return nil, err + } + } + userLogin, err := orm.GetUserLogin(user.UID, user.Account, config.AccountLoginType, tx) + if err != nil { + tx.Rollback() + logger.Error("UpdateLoginInfo get user:%s login error, error msg:%s", user.UID, err.Error()) + return nil, err + } + err = login.CheckSignature(userLogin.LastLoginTime > 0, logger) + if err != nil { + return nil, err + } + if userLogin != nil { + userLogin.LastLoginTime = time.Now().Unix() + err = orm.UpdateUserLogin(user.UID, userLogin, tx) + if err != nil { + tx.Rollback() + logger.Error("UpdateLoginInfo update user:%s login error, error msg:%s", user.UID, err.Error()) + return nil, err + } + } else { + err = orm.CreateUserLogin(&models.UserLogin{ + UID: user.UID, + LastLoginTime: time.Now().Unix(), + LoginId: getLoginId(user, config.AccountLoginType), + LoginType: int(config.AccountLoginType), + }, tx) + if err != nil { + tx.Rollback() + logger.Error("UpdateLoginInfo create user:%s login error, error msg:%s", user.UID, err.Error()) + return nil, err + } + } + err = tx.Commit().Error + if err != nil { + logger.Errorf("SyncUser tx commit error, error msg:%s ", err) + return nil, err + } + return user, nil +} + +func GetUserCount(logger *zap.SugaredLogger) ([]*types.UserCountByType, error) { + resp, err := orm.CountUserByType(core.DB) + if err != nil { + logger.Errorf("Failed to count user by type from db, the error is: %s", err.Error()) + return nil, err + } + return resp, nil +} diff --git a/pkg/microservice/user/server/rest/router.go b/pkg/microservice/user/server/rest/router.go new file mode 100644 index 0000000000000000000000000000000000000000..51139f3a6bddf883dc16c6882df6b72498b8e540 --- /dev/null +++ b/pkg/microservice/user/server/rest/router.go @@ -0,0 +1,35 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rest + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/user/core/handler" +) + +func (s *engine) injectRouterGroup(router *gin.RouterGroup) { + for _, r := range []injector{ + new(handler.Router), + } { + r.Inject(router.Group("/api/v1")) + } +} + +type injector interface { + Inject(router *gin.RouterGroup) +} diff --git a/pkg/microservice/user/server/rest/server.go b/pkg/microservice/user/server/rest/server.go new file mode 100644 index 0000000000000000000000000000000000000000..18843b0a5ead8f194d217a281527d5152e939795 --- /dev/null +++ b/pkg/microservice/user/server/rest/server.go @@ -0,0 +1,76 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rest + +import ( + "net/http" + + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/config" + ginmiddleware "github.com/koderover/zadig/pkg/middleware/gin" + "github.com/koderover/zadig/pkg/tool/log" +) + +type engine struct { + *gin.Engine + + mode string +} + +func NewEngine() *engine { + s := &engine{mode: config.Mode()} + + gin.SetMode(s.mode) + + s.injectMiddlewares() + s.injectRouters() + + return s +} + +func (s *engine) injectMiddlewares() { + g := gin.New() + defer func() { + s.Engine = g + }() + + if s.mode == gin.TestMode { + return + } + g.Use(ginmiddleware.Response()) + g.Use(ginmiddleware.RequestID()) + g.Use(ginmiddleware.RequestLog(log.NewFileLogger(config.RequestLogFile()))) + g.Use(gin.Recovery()) +} + +func (s *engine) injectRouters() { + g := s.Engine + + g.NoRoute(func(c *gin.Context) { + c.String(http.StatusNotFound, "Invalid path: %s", c.Request.URL.Path) + }) + g.HandleMethodNotAllowed = true + g.NoMethod(func(c *gin.Context) { + c.String(http.StatusMethodNotAllowed, "Method not allowed: %s %s", c.Request.Method, c.Request.URL.Path) + }) + + apiRouters := g.Group("") + s.injectRouterGroup(apiRouters) + + s.Engine = g +} diff --git a/pkg/microservice/user/server/server.go b/pkg/microservice/user/server/server.go new file mode 100644 index 0000000000000000000000000000000000000000..aa0f55e5dc51bc614d6fe57f1bf620e9918011b4 --- /dev/null +++ b/pkg/microservice/user/server/server.go @@ -0,0 +1,60 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "context" + "net/http" + "time" + + "github.com/koderover/zadig/pkg/microservice/user/core" + "github.com/koderover/zadig/pkg/microservice/user/server/rest" + "github.com/koderover/zadig/pkg/tool/log" +) + +func Serve(ctx context.Context) error { + core.Start(ctx) + defer core.Stop(ctx) + + log.Info("Start user system service") + + engine := rest.NewEngine() + server := &http.Server{Addr: ":80", Handler: engine} + + stopChan := make(chan struct{}) + go func() { + defer close(stopChan) + + <-ctx.Done() + + ctx, cancel := context.WithTimeout(context.TODO(), 5*time.Second) + defer cancel() + + if err := server.Shutdown(ctx); err != nil { + log.Errorf("Failed to stop server, error: %s", err) + } + }() + + if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Errorf("Failed to start http server, error: %s", err) + return err + } + + <-stopChan + + return nil +} diff --git a/pkg/microservice/warpdrive/config/config.go b/pkg/microservice/warpdrive/config/config.go index 2795735f46e0997740541ee678daa26d4f6832f0..9d74ab6d5b442d2f7930db6b2e760b9cad0c2d6f 100644 --- a/pkg/microservice/warpdrive/config/config.go +++ b/pkg/microservice/warpdrive/config/config.go @@ -34,10 +34,6 @@ func NSQLookupAddrs() []string { return strings.Split(viper.GetString(setting.ENVNsqLookupAddrs), ",") } -func PoetryAPIRootKey() string { - return viper.GetString(setting.ENVPoetryAPIRootKey) -} - func ReleaseImageTimeout() string { return viper.GetString(setting.ReleaseImageTimeout) } @@ -46,14 +42,6 @@ func Home() string { return viper.GetString(setting.Home) } -func DefaultRegistryAddr() string { - return viper.GetString(setting.DefaultRegistryAddr) -} - -func DefaultRegistryAK() string { - return viper.GetString(setting.DefaultRegistryAK) -} - -func DefaultRegistrySK() string { - return viper.GetString(setting.DefaultRegistrySK) +func HomeV2() string { + return viper.GetString(setting.HomeV2) } diff --git a/pkg/microservice/warpdrive/config/const.go b/pkg/microservice/warpdrive/config/const.go index 79acdadf07495d869b93c41bbabe5d120997cb9f..5ee27d05921ea48a0200745f7bff571b9dd3e644 100644 --- a/pkg/microservice/warpdrive/config/const.go +++ b/pkg/microservice/warpdrive/config/const.go @@ -19,24 +19,31 @@ package config type TaskType string const ( - TaskPipeline TaskType = "pipeline" - TaskBuild TaskType = "buildv2" - TaskJenkinsBuild TaskType = "jenkins_build" - TaskArtifact TaskType = "artifact" - TaskDeploy TaskType = "deploy" - TaskTestingV2 TaskType = "testingv2" - TaskDistributeToS3 TaskType = "distribute2kodo" - TaskReleaseImage TaskType = "release_image" - TaskJira TaskType = "jira" - TaskDockerBuild TaskType = "docker_build" - TaskSecurity TaskType = "security" - TaskResetImage TaskType = "reset_image" - TaskDistribute TaskType = "distribute" + TaskPipeline TaskType = "pipeline" + TaskBuild TaskType = "buildv2" + TaskBuildV3 TaskType = "buildv3" + TaskArtifactDeploy TaskType = "artifact_deploy" + TaskJenkinsBuild TaskType = "jenkins_build" + TaskArtifact TaskType = "artifact" + TaskDeploy TaskType = "deploy" + TaskTestingV2 TaskType = "testingv2" + TaskDistributeToS3 TaskType = "distribute2kodo" + TaskReleaseImage TaskType = "release_image" + TaskJira TaskType = "jira" + TaskDockerBuild TaskType = "docker_build" + TaskSecurity TaskType = "security" + TaskResetImage TaskType = "reset_image" + TaskDistribute TaskType = "distribute" + TaskTrigger TaskType = "trigger" + TaskExtension TaskType = "extension" + TaskArtifactPackage TaskType = "artifact_package" + TaskScanning TaskType = "scanning" ) type Status string const ( + StatusInit Status = "init" StatusDisabled Status = "disabled" StatusCreated Status = "created" StatusRunning Status = "running" @@ -49,21 +56,28 @@ const ( StatusQueued Status = "queued" StatusBlocked Status = "blocked" QueueItemPending Status = "pending" + StatusPrepare Status = "prepare" ) type PipelineType string const ( - // SingleType 单服务工作流 + // SingleType single-service workflow SingleType PipelineType = "single" - // WorkflowType 多服务工作流 + // WorkflowType multi-service workflow WorkflowType PipelineType = "workflow" - // FreestyleType 自由编排工作流 + // FreestyleType freeStyle workflow FreestyleType PipelineType = "freestyle" - // TestType 测试 + // TestType testing TestType PipelineType = "test" - // ServiceType 服务 + // ServiceType pipeline ServiceType PipelineType = "service" + // workflowTypeV3 服务 + WorkflowTypeV3 PipelineType = "workflow_v3" + // ArtifactType artifact build + ArtifactType PipelineType = "artifact" + // ScanningType is the type for scanning + ScanningType PipelineType = "scanning" ) type NotifyType int diff --git a/pkg/microservice/warpdrive/core/service/common/dockerhosts.go b/pkg/microservice/warpdrive/core/service/common/dockerhosts.go new file mode 100644 index 0000000000000000000000000000000000000000..33200e3954150ae5ca30773f15eead8769635b84 --- /dev/null +++ b/pkg/microservice/warpdrive/core/service/common/dockerhosts.go @@ -0,0 +1,201 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +import ( + "context" + "fmt" + "math/rand" + "sync" + "time" + + "github.com/buraksezer/consistent" + "github.com/cespare/xxhash" + "go.uber.org/zap" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" +) + +var once sync.Once + +type Member string + +func (m Member) String() string { + return string(m) +} + +type hasher struct{} + +func (h hasher) Sum64(data []byte) uint64 { + return xxhash.Sum64(data) +} + +type ClusterID string + +type DockerHostsI interface { + GetBestHost(ClusterID, string) string + + Sync() +} + +type dockerhosts struct { + rwLock *sync.RWMutex + store map[ClusterID]*consistent.Consistent + hubServerAddr string + syncInterval time.Duration + + logger *zap.SugaredLogger +} + +var dockerHosts DockerHostsI + +func init() { + rand.Seed(time.Now().UTC().UnixNano()) +} + +func NewDockerHosts(hubServerAddr string, logger *zap.SugaredLogger) DockerHostsI { + once.Do(func() { + dockerHosts = &dockerhosts{ + rwLock: &sync.RWMutex{}, + store: map[ClusterID]*consistent.Consistent{}, + hubServerAddr: hubServerAddr, + logger: logger, + } + + go dockerHosts.Sync() + }) + + return dockerHosts +} + +func (d *dockerhosts) GetBestHost(clusterID ClusterID, key string) string { + if d.store[clusterID] == nil { + d.initClusterInfo(clusterID) + } + + member := d.store[clusterID].LocateKey([]byte(key)) + + return member.String() +} + +func (d *dockerhosts) initClusterInfo(clusterID ClusterID) { + d.rwLock.Lock() + defer d.rwLock.Unlock() + + members := d.getDockerHostsSvc(clusterID) + + cfg := consistent.Config{ + PartitionCount: 271, + ReplicationFactor: 20, + Load: 1.25, + Hasher: hasher{}, + } + + d.store[clusterID] = consistent.New(members, cfg) +} + +func (d *dockerhosts) getDockerHostsSvc(clusterID ClusterID) []consistent.Member { + ns := config.Namespace() + if string(clusterID) != setting.LocalClusterID { + ns = setting.AttachedClusterNamespace + } + + kclient, err := kubeclient.GetKubeClient(d.hubServerAddr, string(clusterID)) + if err != nil { + d.logger.Warnf("Failed to get kubeclient for cluster %q: %s. Try to use default dockerhosts.", clusterID, err) + return d.getDefaultDockerHosts() + } + + dindSts := &appsv1.StatefulSet{} + err = kclient.Get(context.TODO(), client.ObjectKey{ + Name: "dind", + Namespace: ns, + }, dindSts) + if err != nil { + d.logger.Warnf("Failed to get dind statefuleset in namespace %q of cluster %q: %s", ns, clusterID, err) + return d.getDefaultDockerHosts() + } + + members := []consistent.Member{} + for i := 0; i < int(*dindSts.Spec.Replicas); i++ { + members = append(members, d.genDindAddr(i)) + } + + return members +} + +func (d *dockerhosts) getDefaultDockerHosts() []consistent.Member { + return []consistent.Member{d.genDindAddr(0)} +} + +func (d *dockerhosts) genDindAddr(idx int) Member { + return Member(fmt.Sprintf("tcp://dind-%d.dind:2375", idx)) +} + +func (d *dockerhosts) Sync() { + d.logger.Info("Begin to sync") + + wait.Forever(func() { + d.rwLock.Lock() + defer d.rwLock.Unlock() + + for clusterID, consistentHash := range d.store { + currentMembers := d.getDockerHostsSvc(clusterID) + oldMembers := consistentHash.GetMembers() + + d.logger.Infof("Cluster: %q. Current Members: %d. Old Members: %d", clusterID, len(currentMembers), len(oldMembers)) + + addedMembers, deletedMembers := d.diffMembers(oldMembers, currentMembers) + for _, member := range addedMembers { + consistentHash.Add(member) + } + for _, member := range deletedMembers { + consistentHash.Remove(member.String()) + } + } + }, 3*time.Minute) +} + +func (d *dockerhosts) diffMembers(old, current []consistent.Member) (added, deleted []consistent.Member) { + curMap := make(map[consistent.Member]struct{}, len(current)) + for _, member := range current { + curMap[member] = struct{}{} + } + + oldMap := make(map[consistent.Member]struct{}, len(old)) + for _, member := range old { + oldMap[member] = struct{}{} + } + + for _, member := range old { + if _, found := curMap[member]; !found { + deleted = append(deleted, member) + } + } + for _, member := range current { + if _, found := oldMap[member]; !found { + added = append(added, member) + } + } + + return added, deleted +} diff --git a/pkg/microservice/warpdrive/core/service/common/dockerhosts_test.go b/pkg/microservice/warpdrive/core/service/common/dockerhosts_test.go new file mode 100644 index 0000000000000000000000000000000000000000..21cf2685c71cf5ac7aad3b090260980fd40f568c --- /dev/null +++ b/pkg/microservice/warpdrive/core/service/common/dockerhosts_test.go @@ -0,0 +1,50 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +import ( + "math/rand" + "testing" + "time" + + "go.uber.org/zap" +) + +func init() { + rand.Seed(time.Now().UTC().UnixNano()) +} + +func TestNewDockerHosts(t *testing.T) { + hubServerAddr := "hub-server.zadig" + logger, _ := zap.NewProduction() + dockerhosts := NewDockerHosts(hubServerAddr, logger.Sugar()) + + clusters := []ClusterID{ClusterID("local"), ClusterID("worker0"), ClusterID("worker1")} + svcs := []string{"svc0", "svc1", "svc2", "svc3", "svc4"} + + for i := 0; i < len(clusters); i++ { + t.Logf("In Cluster %q\n", clusters[i]) + + for j := 0; j < 10; j++ { + svc := svcs[rand.Intn(len(svcs))] + host := dockerhosts.GetBestHost(clusters[i], svc) + t.Logf("\tHost for svc %q: %q\n", svc, host) + } + + t.Log("\n") + } +} diff --git a/pkg/microservice/warpdrive/core/service/common/types.go b/pkg/microservice/warpdrive/core/service/common/types.go new file mode 100644 index 0000000000000000000000000000000000000000..953c10c0e565afde34d7a473fe467fbb52238787 --- /dev/null +++ b/pkg/microservice/warpdrive/core/service/common/types.go @@ -0,0 +1,29 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +import "github.com/koderover/zadig/pkg/microservice/warpdrive/config" + +type Stage struct { + //Note: The same stage cannot temporarily run different types of tasks + TaskType config.TaskType `bson:"type" json:"type"` + Status config.Status `bson:"status" json:"status"` + RunParallel bool `bson:"run_parallel" json:"run_parallel"` + Desc string `bson:"desc,omitempty" json:"desc,omitempty"` + SubTasks map[string]map[string]interface{} `bson:"sub_tasks" json:"sub_tasks"` + AfterAll bool `bson:"after_all" json:"after_all"` +} diff --git a/pkg/microservice/warpdrive/core/service/taskcontroller/interface.go b/pkg/microservice/warpdrive/core/service/taskcontroller/interface.go new file mode 100644 index 0000000000000000000000000000000000000000..bfa6caaf3a1be9fed6604945f7bc603362392de1 --- /dev/null +++ b/pkg/microservice/warpdrive/core/service/taskcontroller/interface.go @@ -0,0 +1,27 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package taskcontroller + +import "context" + +type ControllerI interface { + // Init initializes the controller. + Init(ctx context.Context) error + + // Stop stops process logics. + Stop(ctx context.Context) error +} diff --git a/pkg/microservice/warpdrive/core/service/taskcontroller/task_controller.go b/pkg/microservice/warpdrive/core/service/taskcontroller/task_controller.go index 67ad5c1abcaa67a6c3675e578a376b8c3db686c0..6a04745df09aa05f9d492528ea292482de616123 100644 --- a/pkg/microservice/warpdrive/core/service/taskcontroller/task_controller.go +++ b/pkg/microservice/warpdrive/core/service/taskcontroller/task_controller.go @@ -31,47 +31,57 @@ import ( "github.com/koderover/zadig/pkg/tool/nsqcli" ) -func InitTaskController(ctx context.Context) error { +type controller struct { + consumers []*nsq.Consumer + producers []*nsq.Producer +} + +func NewController() ControllerI { + return &controller{ + consumers: []*nsq.Consumer{}, + producers: []*nsq.Producer{}, + } +} + +func (c *controller) Init(ctx context.Context) error { go func() { if err := client.Start(ctx); err != nil { panic(err) } }() - //初始化nsq cfg := nsq.NewConfig() - // 注意 WD_POD_NAME 必须使用 Downward API 配置环境变量 cfg.UserAgent = config.WarpDrivePodName() - // FIXME: Min: FIX MAGIC NUMBER cfg.MaxAttempts = 50 cfg.LookupdPollInterval = 1 * time.Second - //nsqd 和服务起在一起 FIXME: Min: FIX MAGIC ADDR + cfg.MsgTimeout = 1 * time.Minute nsqClient := nsqcli.NewNsqClient(config.NSQLookupAddrs(), "127.0.0.1:4151") - //Process topic processor, err := nsq.NewConsumer(setting.TopicProcess, "process", cfg) if err != nil { return fmt.Errorf("init nsq processor error: %v", err) } + processor.SetLogger(log.New(os.Stdout, "nsq consumer:", 0), nsq.LogLevelError) + c.consumers = append(c.consumers, processor) - //Cancel topic - //监听不同的channel,确保取消消息到每一个wd canceller, err := nsq.NewConsumer(setting.TopicCancel, cfg.UserAgent, cfg) if err != nil { return fmt.Errorf("init nsq canceller error: %v", err) } + canceller.SetLogger(log.New(os.Stdout, "nsq consumer:", 0), nsq.LogLevelError) + c.consumers = append(c.consumers, canceller) - //Sender nsqdAddr := "127.0.0.1:4150" sender, err := nsq.NewProducer(nsqdAddr, cfg) if err != nil { return fmt.Errorf("init nsq sender error: %v", err) } + sender.SetLogger(log.New(os.Stdout, "nsq producer:", 0), nsq.LogLevelError) + c.producers = append(c.producers, sender) - // 初始化nsq topic err = nsqClient.EnsureNsqdTopics([]string{setting.TopicAck, setting.TopicItReport, setting.TopicNotification}) if err != nil { return fmt.Errorf("ensure nsq topic error: %v", err) @@ -80,22 +90,31 @@ func InitTaskController(ctx context.Context) error { execHandler := &ExecHandler{ Sender: sender, } - processor.AddHandler(execHandler) - - //Add task plugin initiators to exec Handler + // Add task plugin initiators to exec Handler. initTaskPlugins(execHandler) cancelHandler := &CancelHandler{} - canceller.AddHandler(cancelHandler) if err := processor.ConnectToNSQLookupds(config.NSQLookupAddrs()); err != nil { return fmt.Errorf("processor could not connect to %v", config.NSQLookupAddrs()) } - if err := canceller.ConnectToNSQLookupds(config.NSQLookupAddrs()); err != nil { return fmt.Errorf("canceller could not connect to %v", config.NSQLookupAddrs()) } + + return nil +} + +func (c *controller) Stop(ctx context.Context) error { + for _, consumer := range c.consumers { + consumer.Stop() + } + + for _, producer := range c.producers { + producer.Stop() + } + return nil } diff --git a/pkg/microservice/warpdrive/core/service/taskcontroller/task_handler.go b/pkg/microservice/warpdrive/core/service/taskcontroller/task_handler.go index 301be7bf65044b4a5752bd43d188703c1fc8101f..c0ecadb8e60b6c11e37a9efbc94297d9088cb4e7 100644 --- a/pkg/microservice/warpdrive/core/service/taskcontroller/task_handler.go +++ b/pkg/microservice/warpdrive/core/service/taskcontroller/task_handler.go @@ -26,8 +26,10 @@ import ( "github.com/nsqio/go-nsq" uuid "github.com/satori/go.uuid" "go.uber.org/zap" + "k8s.io/apimachinery/pkg/util/sets" "github.com/koderover/zadig/pkg/microservice/warpdrive/config" + "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/common" plugins "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/taskplugin" "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types" "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types/task" @@ -45,7 +47,12 @@ var ( xl *zap.SugaredLogger ) -// ExecHandler ... +// TODO: Leave the logic here until we know why it exists. +var durationBeforeNextTask = 10 * time.Second + +// Note: `durationTouchMsg` is used to emit `TOUCH` cmd and it should smaller than `durationBeforeNextTask`. +var durationTouchMsg = 5 * time.Second + // Sender: sender to send ack/notification // TaskPlugins: registered task plugin initiators to initiate specific plugin to execute task type ExecHandler struct { @@ -53,41 +60,49 @@ type ExecHandler struct { TaskPlugins map[config.TaskType]plugins.Initiator } -// CancelHandler ... type CancelHandler struct{} -// HandleMessage ... // Message handler to handle task execution message func (h *ExecHandler) HandleMessage(message *nsq.Message) error { defer func() { // 每次处理完消息, 等待一段时间不处理新消息 - time.Sleep(time.Second * 10) + time.Sleep(durationBeforeNextTask) }() xl = log.SugaredLogger() - // 如果存在运行中的 PipelineTask, 则重新requeue pipeline task - // task处理逻辑全部放在requeue之后，防止requeue影响正在运行的task - if pipelineTask != nil { - xl.Infof("warpdrive instance have one running pipeline task %s:%d", pipelineTask.PipelineName, pipelineTask.TaskID) - message.Requeue(time.Millisecond * 100) - return nil - } - // 获取 PipelineTask 内容 if err := json.Unmarshal(message.Body, &pipelineTask); err != nil { xl.Errorf("unmarshal PipelineTask error: %v", err) return nil } - xl.Infof("receiving pipeline task %s:%d message", pipelineTask.PipelineName, pipelineTask.TaskID) + taskName := fmt.Sprintf("%s:%d", pipelineTask.PipelineName, pipelineTask.TaskID) + xl.Infof("Receiving pipeline task %s message", taskName) - // xl - global logger xl = Logger(pipelineTask) - - // 初始化 Context, CancelFunc, PipelineTask ctx, cancel = context.WithCancel(context.Background()) - go h.runPipelineTask(ctx, cancel, xl) + go func(ctx context.Context, taskName string) { + for { + select { + case <-ctx.Done(): + xl.Infof("Pipeline task %q has been canceled. Exit.", taskName) + return + case <-time.After(durationTouchMsg): + if pipelineTask == nil { + xl.Infof("Pipeline task %q has completed. Exit.", taskName) + return + } + + xl.Infof("After %s, touch message %q.", durationTouchMsg.String(), taskName) + message.Touch() + } + } + }(ctx, taskName) + + h.runPipelineTask(ctx, cancel, xl) + + // Note: If returning `nil`, we emit `FIN` cmd to nsq indicating that the messsage has been processed succefully. return nil } @@ -128,7 +143,9 @@ func (h *ExecHandler) runPipelineTask(ctx context.Context, cancel context.Cancel return } - // 选取当前最空闲的dockerhost + // Deprecated. + // Note: This logic is reserved for compatibility with plugins other than build. This logic can be removed if we have understood all of the plugins. + // For attached clusters, this logic is wrong because it still deals with dind in the local cluster. dockerHost, err := plugins.GetBestDockerHost(pipelineTask.ConfigPayload.Docker.HostList, string(pipelineTask.Type), pipelineTask.ConfigPayload.Build.KubeNamespace, xl) if err != nil { errMsg := fmt.Sprintf("[%s]Cannot find docker host: %v", pipelineTask.PipelineName, err) @@ -167,7 +184,6 @@ func (h *ExecHandler) runPipelineTask(ctx context.Context, cancel context.Cancel // Return 之前会执行defer内容，更新pipeline end time, 发送ACK，发送notification } -// HandleMessage ... func (h *CancelHandler) HandleMessage(message *nsq.Message) error { xl = Logger(pipelineTask) @@ -251,6 +267,7 @@ func (h *ExecHandler) SendNotification() { Status: config.Status(pipelineTask.Status), TeamName: pipelineTask.TeamName, Type: pipelineTask.Type, + Stages: pipelineTask.Stages, }, CreateTime: time.Now().Unix(), IsRead: false, @@ -268,7 +285,7 @@ func (h *ExecHandler) SendNotification() { } } -func (h *ExecHandler) runStage(stagePosition int, stage *task.Stage) { +func (h *ExecHandler) runStage(stagePosition int, stage *common.Stage, concurrency int64) { xl.Infof("start to execute pipeline stage: %s at position: %d", stage.TaskType, stagePosition) pluginInitiator, ok := h.TaskPlugins[stage.TaskType] if !ok { @@ -285,9 +302,8 @@ func (h *ExecHandler) runStage(stagePosition int, stage *task.Stage) { // Default worker concurrency is 1, run tasks sequentially var workerConcurrency = 1 if runParallel { - // MaxWorkerInParallel is 5 for now - if len(stage.SubTasks) > maxWorkerInParallel { - workerConcurrency = maxWorkerInParallel + if len(stage.SubTasks) > int(concurrency) { + workerConcurrency = int(concurrency) } else { workerConcurrency = len(stage.SubTasks) } @@ -297,30 +313,68 @@ func (h *ExecHandler) runStage(stagePosition int, stage *task.Stage) { // Task is struct for worker var tasks []*Task + //tasks been preprocessed, map[serviceName]=[]Tasks + pluginsByService := make(map[string]*plugins.HelmDeployTaskPlugin) + // helm deploy plugins map[fullServiceName]=>HelmDeployPlugin + helmDeployPlugins := make(map[string]*plugins.HelmDeployTaskPlugin) + + // preprocess subTasks, make batchTask with multiple subTasks + // eg: multiple deploys of same helm chart + if stage.TaskType == config.TaskDeploy || stage.TaskType == config.TaskResetImage { + for fullServiceName, subTask := range stage.SubTasks { + deployTask, err := plugins.ToDeployTask(subTask) + if err != nil { + xl.Errorf("failed to get deplot task, err: %s", err) + continue + } + if deployTask.ServiceType != setting.HelmDeployType { + continue + } + workerConcurrency = 1 + pluginInstance := plugins.InitializeHelmDeployTaskPlugin(config.TaskDeploy) + pluginInstance.Task = deployTask + if _, ok := pluginsByService[deployTask.ServiceName]; !ok { + pluginsByService[deployTask.ServiceName] = pluginInstance + } + helmDeployPlugins[fullServiceName] = pluginInstance + } + } // 每个SubTask会initiate一个plugin instance来执行 + preProcessedServices := sets.NewString() for serviceName, subTask := range stage.SubTasks { + if deployPlugin, ok := helmDeployPlugins[serviceName]; ok { + svcName := deployPlugin.Task.ServiceName + pluginsByService[svcName].ContentPlugins = append(pluginsByService[svcName].ContentPlugins, deployPlugin) + if !preProcessedServices.Has(svcName) { + xl.Infof("new batch sub task of service name: %s, type: %s", serviceName, stage.TaskType) + batchTask := NewTask(ctx, h.executeTask, pluginsByService[svcName], subTask, stagePosition, serviceName, xl) + tasks = append(tasks, batchTask) + } + preProcessedServices.Insert(svcName) + continue + } + var pluginInstance plugins.TaskPlugin xl.Infof("new sub task of service name: %s, type: %s", serviceName, stage.TaskType) pluginInstance = pluginInitiator(stage.TaskType) - //xl.Errorf("%v", ctx.Value(CtxKeyBuildInfos)) - tasks = append(tasks, NewTask(ctx, h.executeTask, pluginInstance, subTask, stagePosition, serviceName, xl)) - } - // 判断subTask是否是deploy，如果是的话判断是否是helm类型的服务， - //todo helm类型的服务的部署暂时只支持串行执行 - for _, subTask := range stage.SubTasks { - if deploy, err := plugins.ToDeployTask(subTask); err == nil { - if deploy.ServiceType == "helm" { - workerConcurrency = 1 - break - } - } + taskObj := NewTask(ctx, h.executeTask, pluginInstance, subTask, stagePosition, serviceName, xl) + tasks = append(tasks, taskObj) } // 设置WorkPool来控制最大并发数和并发执行 workerPool := NewPool(tasks, workerConcurrency) // 发起workerConcurrency个并发执行，等待所有Task执行完成并返回 workerPool.Run() + + // set related task status + for _, helmDeployPlugin := range helmDeployPlugins { + if len(helmDeployPlugin.ContentPlugins) == 0 { + continue + } + updatePluginSubTask(helmDeployPlugin, pipelineTask, stagePosition, helmDeployPlugin.Task.ContainerName, xl) + } + // Worker is completed xl.Info("execution completed of subtasks in stage") stageStatus := getStageStatus(workerPool.Tasks, xl) @@ -337,7 +391,7 @@ func (h *ExecHandler) runStage(stagePosition int, stage *task.Stage) { func (h *ExecHandler) execute(ctx context.Context, pipelineTask *task.Task, pipelineCtx *task.PipelineCtx, xl *zap.SugaredLogger) { xl.Info("start pipeline task executor...") // 如果是pipeline 1.0， 先将subtasks进行transform，转化为stages结构 - if pipelineTask.Type == config.SingleType || pipelineTask.Type == "" { + if pipelineTask.Type == config.SingleType || pipelineTask.Type == "" || pipelineTask.Type == config.WorkflowTypeV3 { err := transformToStages(pipelineTask, xl) // 初始化出错时，直接返回pipeline状态错误 if err != nil { @@ -347,20 +401,48 @@ func (h *ExecHandler) execute(ctx context.Context, pipelineTask *task.Task, pipe } } - // Stage之间仅支持串行 + // Only serial is supported between stages + // If the stage status is StatusFailed/StatusCancelled/StatusTimeout, other than the extension stage will not be executed + isSkip := false for stagePosition, stage := range pipelineTask.Stages { - if !stage.AfterAll { - h.runStage(stagePosition, stage) - // 如果一个Stage执行失败了，跳出执行循环，并且更新pipelinetask状态为失败，发送ACK，并返回 - if stage.Status == config.StatusFailed || stage.Status == config.StatusCancelled || stage.Status == config.StatusTimeout { - break - } + if stage.AfterAll { + continue + } + + if !isSkip || stage.TaskType == config.TaskExtension { + h.runStage(stagePosition, stage, pipelineTask.ConfigPayload.BuildConcurrency) + } + + if stage.Status == config.StatusFailed || stage.Status == config.StatusCancelled || stage.Status == config.StatusTimeout { + isSkip = true + continue } } + updatePipelineStatus(pipelineTask, xl) + deployStageStatus := config.StatusInit + testStageStatus := config.StatusInit for stagePosition, stage := range pipelineTask.Stages { + if stage.TaskType == config.TaskDeploy || stage.TaskType == config.TaskArtifact { + deployStageStatus = stage.Status + } else if stage.TaskType == config.TaskTestingV2 { + testStageStatus = stage.Status + } if stage.AfterAll { - h.runStage(stagePosition, stage) + if stage.TaskType == config.TaskResetImage { + switch pipelineTask.ResetImagePolicy { + case setting.ResetImagePolicyTaskCompleted, setting.ResetImagePolicyTaskCompletedOrder: + case setting.ResetImagePolicyDeployFailed: + if deployStageStatus == config.StatusInit || (deployStageStatus != config.StatusFailed && deployStageStatus != config.StatusCancelled && deployStageStatus != config.StatusTimeout) { + continue + } + case setting.ResetImagePolicyTestFailed: + if testStageStatus == config.StatusInit || (testStageStatus != config.StatusFailed && testStageStatus != config.StatusCancelled && testStageStatus != config.StatusTimeout) { + continue + } + } + } + h.runStage(stagePosition, stage, pipelineTask.ConfigPayload.BuildConcurrency) } } @@ -411,6 +493,15 @@ func (h *ExecHandler) executeTask(taskCtx context.Context, plugin plugins.TaskPl } else if pipelineTask.Type == config.ServiceType { fileName = strings.Replace(strings.ToLower(fmt.Sprintf("%s-%s-%d-%s-%s", config.ServiceType, pipelineTask.PipelineName, pipelineTask.TaskID, plugin.Type(), servicename)), "_", "-", -1) + } else if pipelineTask.Type == config.WorkflowTypeV3 { + fileName = strings.Replace(strings.ToLower(fmt.Sprintf("%s-%s-%d-%s-%s", config.WorkflowTypeV3, pipelineTask.PipelineName, pipelineTask.TaskID, plugin.Type(), fmt.Sprintf("%s-job", pipelineTask.PipelineName))), + "_", "-", -1) + } else if pipelineTask.Type == config.ArtifactType { + fileName = strings.Replace(strings.ToLower(fmt.Sprintf("%s-%s-%d-%s", config.ArtifactType, pipelineTask.PipelineName, pipelineTask.TaskID, plugin.Type())), + "_", "-", -1) + } else if pipelineTask.Type == config.ScanningType { + fileName = strings.Replace(strings.ToLower(fmt.Sprintf("%s-%s-%d-%s", config.ScanningType, pipelineTask.PipelineName, pipelineTask.TaskID, plugin.Type())), + "_", "-", -1) } plugin.Init(jobName, fileName, xl) @@ -435,7 +526,12 @@ func (h *ExecHandler) executeTask(taskCtx context.Context, plugin plugins.TaskPl } // 设置 SubTask 初始状态 - plugin.SetStatus(config.StatusRunning) + switch plugin.Type() { + case config.TaskBuild, config.TaskTestingV2: + plugin.SetStatus(config.StatusPrepare) + default: + plugin.SetStatus(config.StatusRunning) + } // 设置 SubTask 开始时间 plugin.SetStartTime() @@ -443,18 +539,18 @@ func (h *ExecHandler) executeTask(taskCtx context.Context, plugin plugins.TaskPl // 清除上一次错误信息 plugin.ResetError() - updatePipelineSubTask(plugin.GetTask(), pipelineTask, pos, servicename, xl) + updatePluginSubTask(plugin, pipelineTask, pos, servicename, xl) h.SendAck() plugin.SetAckFunc(func() { - updatePipelineSubTask(plugin.GetTask(), pipelineTask, pos, servicename, xl) + updatePluginSubTask(plugin, pipelineTask, pos, servicename, xl) h.SendAck() }) xl.Info("start to call plugin.Run") // 如果是并行跑，用servicename来区分不同的workspace runCtx := *pipelineCtx - if pipelineTask.Type == config.WorkflowType { + if pipelineTask.Type == config.WorkflowType || pipelineTask.Type == config.WorkflowTypeV3 { runCtx.Workspace = fmt.Sprintf("%s/%s", pipelineCtx.Workspace, servicename) } // 运行 SubTask, 如果需要异步，请在方法内实现 @@ -463,15 +559,18 @@ func (h *ExecHandler) executeTask(taskCtx context.Context, plugin plugins.TaskPl // 如果 SubTask 执行失败, 则不继续执行, 发送 Task 失败执行结果 // Failed, Timeout, Cancelled if plugin.IsTaskFailed() { + plugin.Complete(ctx, pipelineTask, servicename) + xl.Infof("task status: %s", plugin.Status()) + plugin.SetEndTime() - updatePipelineSubTask(plugin.GetTask(), pipelineTask, pos, servicename, xl) + updatePluginSubTask(plugin, pipelineTask, pos, servicename, xl) return plugin.Status(), fmt.Errorf("pipeline task failed: task_handler:308") } xl.Infof("task status: %s", plugin.Status()) // 等待完成前, 更新 SubTask 执行结果到 PipelineTask - updatePipelineSubTask(plugin.GetTask(), pipelineTask, pos, servicename, xl) + updatePluginSubTask(plugin, pipelineTask, pos, servicename, xl) h.SendAck() // 等待 SubTask 结束 @@ -488,7 +587,7 @@ func (h *ExecHandler) executeTask(taskCtx context.Context, plugin plugins.TaskPl } // 更新 SubTask 执行结果到 PipelineTask plugin.SetEndTime() - updatePipelineSubTask(plugin.GetTask(), pipelineTask, pos, servicename, xl) + updatePluginSubTask(plugin, pipelineTask, pos, servicename, xl) h.SendAck() xl.Infof("end sub task [%s:%s]", plugin.Type(), plugin.Status()) @@ -496,7 +595,6 @@ func (h *ExecHandler) executeTask(taskCtx context.Context, plugin plugins.TaskPl } func Logger(pipelineTask *task.Task) *zap.SugaredLogger { - // 初始化Logger l := log.Logger() if pipelineTask != nil { l.With(zap.String(setting.RequestID, pipelineTask.ReqID)) diff --git a/pkg/microservice/warpdrive/core/service/taskcontroller/task_helper.go b/pkg/microservice/warpdrive/core/service/taskcontroller/task_helper.go index 867b82e336686a67232365bf6609fef68dd905c5..6076af284fe35d42007dfd7ced1bb07ae7e80620 100644 --- a/pkg/microservice/warpdrive/core/service/taskcontroller/task_helper.go +++ b/pkg/microservice/warpdrive/core/service/taskcontroller/task_helper.go @@ -29,6 +29,7 @@ import ( configbase "github.com/koderover/zadig/pkg/config" "github.com/koderover/zadig/pkg/microservice/warpdrive/config" + "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/common" plugins "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/taskplugin" "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/taskplugin/github" "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/taskplugin/s3" @@ -44,7 +45,7 @@ import ( // 1. 转换经过序列化的 SubTasks 到 Stages // Stage Map: *Stage -> map [service->subtask] func transformToStages(pipelineTask *task.Task, xl *zap.SugaredLogger) error { - var pipelineStages []*task.Stage + var pipelineStages []*common.Stage // 工作流1.0，单服务工作流，SubTasks一维数组 // Transform into stages and assign to stages // Task的数据结构中如果没有赋值Type，也按照1.0处理 @@ -57,7 +58,7 @@ func transformToStages(pipelineTask *task.Task, xl *zap.SugaredLogger) error { } // Pipeline 1.0中的一个Subtask对应到Pipeline 2.0中的一个Stage // Stage中subtasks仅有一个subtask, key为service_name - stage := &task.Stage{ + stage := &common.Stage{ TaskType: subTaskPreview.TaskType, // Pipeline 1.0中，每个type subtask只有一个，不存在并行执行 RunParallel: false, @@ -91,6 +92,17 @@ func initPipelineTask(pipelineTask *task.Task, xl *zap.SugaredLogger) { } } +func updatePluginSubTask(plugin plugins.TaskPlugin, pipelineTask *task.Task, pos int, servicename string, xl *zap.SugaredLogger) { + helmDeployPlugin, ok := plugin.(*plugins.HelmDeployTaskPlugin) + if ok && helmDeployPlugin != nil { + for _, p := range helmDeployPlugin.ContentPlugins { + updatePipelineSubTask(p.Task, pipelineTask, pos, p.Task.ContainerName, xl) + } + } else { + updatePipelineSubTask(plugin.GetTask(), pipelineTask, pos, servicename, xl) + } +} + // Notes: // 1.0中pos代表subtasks位置 // 2.0中pos代表stages位置 @@ -121,10 +133,10 @@ func updatePipelineSubTask(t interface{}, pipelineTask *task.Task, pos int, serv // 同时更新stages // TODO: 完善Stage其他字段 if len(pipelineTask.Stages) == 0 { - pipelineTask.Stages = make([]*task.Stage, len(pipelineTask.SubTasks)) + pipelineTask.Stages = make([]*common.Stage, len(pipelineTask.SubTasks)) } if pipelineTask.Stages[pos] == nil { - pipelineTask.Stages[pos] = &task.Stage{} + pipelineTask.Stages[pos] = &common.Stage{} } pipelineTask.Stages[pos].SubTasks = map[string]map[string]interface{}{servicename: subTask} } else if pipelineTask.Type == config.WorkflowType { @@ -136,6 +148,12 @@ func updatePipelineSubTask(t interface{}, pipelineTask *task.Task, pos int, serv } else if pipelineTask.Type == config.ServiceType { xl.Info("pipeline type is service type: pipeline 3.0") pipelineTask.Stages[pos].SubTasks[servicename] = subTask + } else if pipelineTask.Type == config.WorkflowTypeV3 { + xl.Info("pipeline type is workflow type: pipeline 3.0") + pipelineTask.Stages[pos].SubTasks[servicename] = subTask + } else if pipelineTask.Type == config.ArtifactType { + xl.Info("pipeline type is artifact-package type: pipeline 3.0") + pipelineTask.Stages[pos].SubTasks[servicename] = subTask } } @@ -144,7 +162,7 @@ func updatePipelineSubTask(t interface{}, pipelineTask *task.Task, pos int, serv func updatePipelineStageStatus(stageStatus config.Status, pipelineTask *task.Task, pos int, xl *zap.SugaredLogger) { xl.Infof("updating pipeline task, stage status: %s, stage position: %d", stageStatus, pos) if pipelineTask.Stages[pos] == nil { - pipelineTask.Stages[pos] = &task.Stage{} + pipelineTask.Stages[pos] = &common.Stage{} } pipelineTask.Stages[pos].Status = stageStatus } @@ -538,16 +556,22 @@ func getSubTaskTypeAndIsRestart(subTask map[string]interface{}) bool { func initTaskPlugins(execHandler *ExecHandler) { pluginConf := map[config.TaskType]plugins.Initiator{ - config.TaskJira: plugins.InitializeJiraTaskPlugin, - config.TaskBuild: plugins.InitializeBuildTaskPlugin, - config.TaskJenkinsBuild: plugins.InitializeJenkinsBuildPlugin, - config.TaskDockerBuild: plugins.InitializeDockerBuildTaskPlugin, - config.TaskDeploy: plugins.InitializeDeployTaskPlugin, - config.TaskTestingV2: plugins.InitializeTestTaskPlugin, - config.TaskSecurity: plugins.InitializeSecurityPlugin, - config.TaskReleaseImage: plugins.InitializeReleaseImagePlugin, - config.TaskDistributeToS3: plugins.InitializeDistribute2S3TaskPlugin, - config.TaskResetImage: plugins.InitializeDeployTaskPlugin, + config.TaskJira: plugins.InitializeJiraTaskPlugin, + config.TaskBuild: plugins.InitializeBuildTaskPlugin, + config.TaskBuildV3: plugins.InitializeBuildTaskV3Plugin, + config.TaskArtifactDeploy: plugins.InitializeArtifactTaskPlugin, + config.TaskJenkinsBuild: plugins.InitializeJenkinsBuildPlugin, + config.TaskDockerBuild: plugins.InitializeDockerBuildTaskPlugin, + config.TaskDeploy: plugins.InitializeDeployTaskPlugin, + config.TaskTestingV2: plugins.InitializeTestTaskPlugin, + config.TaskSecurity: plugins.InitializeSecurityPlugin, + config.TaskReleaseImage: plugins.InitializeReleaseImagePlugin, + config.TaskDistributeToS3: plugins.InitializeDistribute2S3TaskPlugin, + config.TaskResetImage: plugins.InitializeDeployTaskPlugin, + config.TaskTrigger: plugins.InitializeTriggerTaskPlugin, + config.TaskArtifactPackage: plugins.InitializeArtifactPackagePlugin, + config.TaskExtension: plugins.InitializeExtensionTaskPlugin, + config.TaskScanning: plugins.InitializeScanningTaskPlugin, } for name, pluginInitiator := range pluginConf { registerTaskPlugin(execHandler, name, pluginInitiator) diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/artifact_deploy.go b/pkg/microservice/warpdrive/core/service/taskplugin/artifact_deploy.go new file mode 100644 index 0000000000000000000000000000000000000000..581085f09785f63c11289b40698c9b2bf3b9f156 --- /dev/null +++ b/pkg/microservice/warpdrive/core/service/taskplugin/artifact_deploy.go @@ -0,0 +1,385 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package taskplugin + +import ( + "context" + "fmt" + "math/rand" + "strconv" + "strings" + "time" + + "go.uber.org/zap" + "gopkg.in/yaml.v3" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + zadigconfig "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/warpdrive/config" + "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types/task" + "github.com/koderover/zadig/pkg/setting" + krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client" + "github.com/koderover/zadig/pkg/tool/kube/updater" +) + +const ( + // ArtifactDeployTaskV2Timeout ... + ArtifactDeployTaskV2Timeout = 60 * 60 * 1 // 60 minutes +) + +// InitializeArtifactTaskPlugin to initialize build task plugin, and return reference +func InitializeArtifactTaskPlugin(taskType config.TaskType) TaskPlugin { + return &ArtifactDeployTaskPlugin{ + Name: taskType, + kubeClient: krkubeclient.Client(), + clientset: krkubeclient.Clientset(), + restConfig: krkubeclient.RESTConfig(), + } +} + +// BuildTaskPlugin is Plugin, name should be compatible with task type +type ArtifactDeployTaskPlugin struct { + Name config.TaskType + KubeNamespace string + JobName string + FileName string + kubeClient client.Client + clientset kubernetes.Interface + restConfig *rest.Config + Task *task.Build + Log *zap.SugaredLogger + + ack func() +} + +func (p *ArtifactDeployTaskPlugin) SetAckFunc(ack func()) { + p.ack = ack +} + +// Init ... +func (p *ArtifactDeployTaskPlugin) Init(jobname, filename string, xl *zap.SugaredLogger) { + p.JobName = jobname + p.Log = xl + p.FileName = filename +} + +func (p *ArtifactDeployTaskPlugin) Type() config.TaskType { + return p.Name +} + +// Status ... +func (p *ArtifactDeployTaskPlugin) Status() config.Status { + return p.Task.TaskStatus +} + +// SetStatus ... +func (p *ArtifactDeployTaskPlugin) SetStatus(status config.Status) { + p.Task.TaskStatus = status +} + +// TaskTimeout ... +func (p *ArtifactDeployTaskPlugin) TaskTimeout() int { + if p.Task.Timeout == 0 { + p.Task.Timeout = ArtifactDeployTaskV2Timeout + } else { + if !p.Task.IsRestart { + p.Task.Timeout = p.Task.Timeout * 60 + } + } + return p.Task.Timeout +} + +func (p *ArtifactDeployTaskPlugin) SetBuildStatusCompleted(status config.Status) { + p.Task.BuildStatus.Status = status + p.Task.BuildStatus.EndTime = time.Now().Unix() +} + +func (p *ArtifactDeployTaskPlugin) Run(ctx context.Context, pipelineTask *task.Task, pipelineCtx *task.PipelineCtx, serviceName string) { + switch p.Task.ClusterID { + case setting.LocalClusterID: + p.KubeNamespace = zadigconfig.Namespace() + default: + p.KubeNamespace = setting.AttachedClusterNamespace + + crClient, clientset, restConfig, err := GetK8sClients(pipelineTask.ConfigPayload.HubServerAddr, p.Task.ClusterID) + if err != nil { + p.Log.Error(err) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = err.Error() + p.SetBuildStatusCompleted(config.StatusFailed) + return + } + + p.kubeClient = crClient + p.clientset = clientset + p.restConfig = restConfig + } + + envName := pipelineTask.WorkflowArgs.Namespace + envNameVar := &task.KeyVal{Key: "ENV_NAME", Value: envName, IsCredential: false} + p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, envNameVar) + + taskIDVar := &task.KeyVal{Key: "TASK_ID", Value: strconv.FormatInt(pipelineTask.TaskID, 10), IsCredential: false} + p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, taskIDVar) + + privateKeys := sets.String{} + for _, privateKey := range pipelineTask.ConfigPayload.PrivateKeys { + privateKeys.Insert(privateKey.Name) + } + + privateKeysVar := &task.KeyVal{Key: "AGENTS", Value: strings.Join(privateKeys.List(), ","), IsCredential: false} + p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, privateKeysVar) + + // env host ips + for envName, HostIPs := range p.Task.EnvHostInfo { + envHostKeysVar := &task.KeyVal{Key: envName + "_HOST_IPs", Value: strings.Join(HostIPs, ","), IsCredential: false} + p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, envHostKeysVar) + } + + // env host names + for envName, names := range p.Task.EnvHostNames { + envHostKeysVar := &task.KeyVal{Key: envName + "_HOST_NAMEs", Value: strings.Join(names, ","), IsCredential: false} + p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, envHostKeysVar) + } + + // ARTIFACT + if p.Task.ArtifactInfo != nil { + var workspace = "/workspace" + if pipelineTask.ConfigPayload.ClassicBuild { + workspace = pipelineCtx.Workspace + } + pipelineTask.ArtifactInfo = p.Task.ArtifactInfo + artifactKeysVar := &task.KeyVal{Key: "ARTIFACT", Value: fmt.Sprintf("%s/%s", workspace, p.Task.ArtifactInfo.FileName), IsCredential: false} + p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, artifactKeysVar) + } + + p.KubeNamespace = pipelineTask.ConfigPayload.Build.KubeNamespace + + //instantiates variables like ${_BRANCH} ${${REPO_index}_BRANCH} .. + p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, InstantiateBuildSysVariables(&p.Task.JobCtx)...) + + jobCtx := JobCtxBuilder{ + JobName: p.JobName, + PipelineCtx: pipelineCtx, + ArchiveFile: p.Task.JobCtx.PackageFile, + JobCtx: p.Task.JobCtx, + Installs: p.Task.InstallCtx, + } + + if p.Task.BuildStatus == nil { + p.Task.BuildStatus = &task.BuildStatus{} + } + + p.Task.BuildStatus.Status = config.StatusRunning + p.Task.BuildStatus.StartTime = time.Now().Unix() + p.ack() + + jobCtxBytes, err := yaml.Marshal(jobCtx.BuildReaperContext(pipelineTask, serviceName)) + if err != nil { + msg := fmt.Sprintf("cannot reaper.Context data: %v", err) + p.Log.Error(msg) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = msg + p.SetBuildStatusCompleted(config.StatusFailed) + return + } + + jobLabel := &JobLabel{ + PipelineName: pipelineTask.PipelineName, + ServiceName: serviceName, + TaskID: pipelineTask.TaskID, + TaskType: string(p.Type()), + PipelineType: string(pipelineTask.Type), + } + + if err := ensureDeleteConfigMap(p.KubeNamespace, jobLabel, p.kubeClient); err != nil { + p.Log.Error(err) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = err.Error() + p.SetBuildStatusCompleted(config.StatusFailed) + return + } + + if err := createJobConfigMap( + p.KubeNamespace, p.JobName, jobLabel, string(jobCtxBytes), p.kubeClient); err != nil { + msg := fmt.Sprintf("createJobConfigMap error: %v", err) + p.Log.Error(msg) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = msg + p.SetBuildStatusCompleted(config.StatusFailed) + return + } + p.Log.Infof("succeed to create cm for build job %s", p.JobName) + + jobImage := getReaperImage(pipelineTask.ConfigPayload.Release.ReaperImage, p.Task.BuildOS, p.Task.ImageFrom) + + //Resource request default value is LOW + job, err := buildJob(p.Type(), jobImage, p.JobName, serviceName, "", pipelineTask.ConfigPayload.Build.KubeNamespace, p.Task.ResReq, p.Task.ResReqSpec, pipelineCtx, pipelineTask, p.Task.Registries) + if err != nil { + msg := fmt.Sprintf("create build job context error: %v", err) + p.Log.Error(msg) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = msg + p.SetBuildStatusCompleted(config.StatusFailed) + return + } + + job.Namespace = p.KubeNamespace + + if err := ensureDeleteJob(p.KubeNamespace, jobLabel, p.kubeClient); err != nil { + msg := fmt.Sprintf("delete build job error: %v", err) + p.Log.Error(msg) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = msg + p.SetBuildStatusCompleted(config.StatusFailed) + return + } + + // 将集成到KodeRover的私有镜像仓库的访问权限设置到namespace中 + if err := createOrUpdateRegistrySecrets(p.KubeNamespace, pipelineTask.ConfigPayload.RegistryID, p.Task.Registries, p.kubeClient); err != nil { + msg := fmt.Sprintf("create secret error: %v", err) + p.Log.Error(msg) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = msg + p.SetBuildStatusCompleted(config.StatusFailed) + return + } + if err := updater.CreateJob(job, p.kubeClient); err != nil { + msg := fmt.Sprintf("create build job error: %v", err) + p.Log.Error(msg) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = msg + p.SetBuildStatusCompleted(config.StatusFailed) + return + } + p.Log.Infof("succeed to create build job %s", p.JobName) +} + +// Wait ... +func (p *ArtifactDeployTaskPlugin) Wait(ctx context.Context) { + status := waitJobEndWithFile(ctx, p.TaskTimeout(), p.KubeNamespace, p.JobName, true, p.kubeClient, p.clientset, p.restConfig, p.Log) + p.SetBuildStatusCompleted(status) + + if status == config.StatusPassed { + if p.Task.DockerBuildStatus == nil { + p.Task.DockerBuildStatus = &task.DockerBuildStatus{} + } + + p.Task.DockerBuildStatus.StartTime = time.Now().Unix() + p.Task.DockerBuildStatus.Status = config.StatusRunning + p.ack() + + select { + case <-ctx.Done(): + p.Task.DockerBuildStatus.EndTime = time.Now().Unix() + p.Task.DockerBuildStatus.Status = config.StatusCancelled + p.Task.TaskStatus = config.StatusCancelled + p.ack() + return + case <-time.After(time.Duration(rand.Int()%2) * time.Second): + p.Task.DockerBuildStatus.EndTime = time.Now().Unix() + p.Task.DockerBuildStatus.Status = config.StatusPassed + } + } + + p.SetStatus(status) +} + +// Complete ... +func (p *ArtifactDeployTaskPlugin) Complete(ctx context.Context, pipelineTask *task.Task, serviceName string) { + jobLabel := &JobLabel{ + PipelineName: pipelineTask.PipelineName, + ServiceName: serviceName, + TaskID: pipelineTask.TaskID, + TaskType: string(p.Type()), + PipelineType: string(pipelineTask.Type), + } + + // 清理用户取消和超时的任务 + defer func() { + if p.Task.TaskStatus == config.StatusCancelled || p.Task.TaskStatus == config.StatusTimeout { + if err := ensureDeleteJob(p.KubeNamespace, jobLabel, p.kubeClient); err != nil { + p.Log.Error(err) + p.Task.Error = err.Error() + } + return + } + }() + + err := saveContainerLog(pipelineTask, p.KubeNamespace, "", p.FileName, jobLabel, p.kubeClient) + if err != nil { + p.Log.Error(err) + p.Task.Error = err.Error() + return + } + + p.Task.LogFile = p.FileName +} + +// SetTask ... +func (p *ArtifactDeployTaskPlugin) SetTask(t map[string]interface{}) error { + task, err := ToBuildTask(t) + if err != nil { + return err + } + p.Task = task + return nil +} + +// GetTask ... +func (p *ArtifactDeployTaskPlugin) GetTask() interface{} { + return p.Task +} + +// IsTaskDone ... +func (p *ArtifactDeployTaskPlugin) IsTaskDone() bool { + if p.Task.TaskStatus != config.StatusCreated && p.Task.TaskStatus != config.StatusRunning { + return true + } + return false +} + +// IsTaskFailed ... +func (p *ArtifactDeployTaskPlugin) IsTaskFailed() bool { + if p.Task.TaskStatus == config.StatusFailed || p.Task.TaskStatus == config.StatusTimeout || p.Task.TaskStatus == config.StatusCancelled { + return true + } + return false +} + +// SetStartTime ... +func (p *ArtifactDeployTaskPlugin) SetStartTime() { + p.Task.StartTime = time.Now().Unix() +} + +// SetEndTime ... +func (p *ArtifactDeployTaskPlugin) SetEndTime() { + p.Task.EndTime = time.Now().Unix() +} + +// IsTaskEnabled ... +func (p *ArtifactDeployTaskPlugin) IsTaskEnabled() bool { + return p.Task.Enabled +} + +// ResetError ... +func (p *ArtifactDeployTaskPlugin) ResetError() { + p.Task.Error = "" +} diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/artifact_package.go b/pkg/microservice/warpdrive/core/service/taskplugin/artifact_package.go new file mode 100644 index 0000000000000000000000000000000000000000..a2eec1c000d08bc2a822f1c7e0ef903a665eeaa2 --- /dev/null +++ b/pkg/microservice/warpdrive/core/service/taskplugin/artifact_package.go @@ -0,0 +1,422 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package taskplugin + +import ( + "context" + "fmt" + "time" + + "go.uber.org/zap" + "gopkg.in/yaml.v3" + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/koderover/zadig/pkg/microservice/warpdrive/config" + "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types" + "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types/task" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/kube/wrapper" + krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client" + "github.com/koderover/zadig/pkg/tool/kube/getter" + "github.com/koderover/zadig/pkg/tool/kube/podexec" + "github.com/koderover/zadig/pkg/tool/kube/updater" +) + +// InitializeArtifactPackagePlugin to ini +func InitializeArtifactPackagePlugin(taskType config.TaskType) TaskPlugin { + return &ArtifactPackageTaskPlugin{ + Name: taskType, + kubeClient: krkubeclient.Client(), + } +} + +type ArtifactPackageTaskPlugin struct { + Name config.TaskType + KubeNamespace string + JobName string + FileName string + kubeClient client.Client + Task *task.ArtifactPackage + AckFunc func() + Log *zap.SugaredLogger + + RealTimeProgress string +} + +const ( + // ArtifactPackageBuildImageTaskTimeout ... + ArtifactPackageBuildImageTaskTimeout = 60 * 5 // 5 minutes +) + +// Init ... +func (p *ArtifactPackageTaskPlugin) Init(jobname, filename string, xl *zap.SugaredLogger) { + p.JobName = jobname + p.FileName = filename + // SetLogger ... + p.Log = xl +} + +func (p *ArtifactPackageTaskPlugin) SetAckFunc(f func()) { + p.AckFunc = f +} + +// Type ... +func (p *ArtifactPackageTaskPlugin) Type() config.TaskType { + return p.Name +} + +// Status ... +func (p *ArtifactPackageTaskPlugin) Status() config.Status { + return p.Task.TaskStatus +} + +// SetStatus ... +func (p *ArtifactPackageTaskPlugin) SetStatus(status config.Status) { + p.Task.TaskStatus = status +} + +func (p *ArtifactPackageTaskPlugin) SetProgress(progress string) { + p.Task.Progress = progress +} + +// TaskTimeout ... +func (p *ArtifactPackageTaskPlugin) TaskTimeout() int { + if p.Task.Timeout == 0 { + p.Task.Timeout = ArtifactPackageBuildImageTaskTimeout + } + return p.Task.Timeout +} + +func (p *ArtifactPackageTaskPlugin) Run(ctx context.Context, pipelineTask *task.Task, pipelineCtx *task.PipelineCtx, serviceName string) { + p.KubeNamespace = pipelineTask.ConfigPayload.Build.KubeNamespace + + sourceRegistries := make([]*types.DockerRegistry, 0) + for _, registryID := range pipelineTask.ArtifactPackageTaskArgs.SourceRegistries { + if registry, ok := pipelineTask.ConfigPayload.RepoConfigs[registryID]; ok { + sourceRegistries = append(sourceRegistries, &types.DockerRegistry{ + RegistryID: registryID, + Host: registry.RegAddr, + Namespace: registry.Namespace, + UserName: registry.AccessKey, + Password: registry.SecretKey, + }) + } + } + + targetRegistries := make([]*types.DockerRegistry, 0) + for _, registryID := range pipelineTask.ArtifactPackageTaskArgs.TargetRegistries { + if registry, ok := pipelineTask.ConfigPayload.RepoConfigs[registryID]; ok { + targetRegistries = append(targetRegistries, &types.DockerRegistry{ + RegistryID: registryID, + Host: registry.RegAddr, + Namespace: registry.Namespace, + UserName: registry.AccessKey, + Password: registry.SecretKey, + }) + } + } + + jobCtx := &types.ArtifactPackagerContext{ + JobType: setting.BuildChartPackage, + ProgressFile: setting.ProgressFile, + Images: pipelineTask.ArtifactPackageTaskArgs.Images, + SourceRegistries: sourceRegistries, + TargetRegistries: targetRegistries, + } + + jobCtxBytes, err := yaml.Marshal(jobCtx) + if err != nil { + msg := fmt.Sprintf("cannot mashal predetor.Context data: %v", err) + p.Log.Error(msg) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = msg + return + } + + p.Task.Error = "" + + jobLabel := &JobLabel{ + PipelineName: pipelineTask.PipelineName, + TaskID: pipelineTask.TaskID, + TaskType: string(p.Type()), + PipelineType: string(pipelineTask.Type), + } + + if err := ensureDeleteConfigMap(p.KubeNamespace, jobLabel, p.kubeClient); err != nil { + msg := fmt.Sprintf("ensureDeleteConfigMap error: %v", err) + p.Log.Error(msg) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = msg + return + } + + if err := createJobConfigMap( + p.KubeNamespace, p.JobName, jobLabel, string(jobCtxBytes), p.kubeClient); err != nil { + msg := fmt.Sprintf("createJobConfigMap error: %v", err) + p.Log.Error(msg) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = msg + return + } + p.Log.Infof("succeed to create cm for artifact package job %s", p.JobName) + + job, err := buildJob(p.Type(), pipelineTask.ConfigPayload.Release.PackagerImage, p.JobName, serviceName, "", pipelineTask.ConfigPayload.Build.KubeNamespace, setting.MinRequest, setting.LowRequestSpec, pipelineCtx, pipelineTask, []*task.RegistryNamespace{}) + if err != nil { + msg := fmt.Sprintf("create release artifact package job context error: %v", err) + p.Log.Error(msg) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = msg + return + } + + if err := ensureDeleteJob(p.KubeNamespace, jobLabel, p.kubeClient); err != nil { + msg := fmt.Sprintf("delete release artifact package job error: %v", err) + p.Log.Error(msg) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = msg + return + } + + job.Namespace = p.KubeNamespace + if err := updater.CreateJob(job, p.kubeClient); err != nil { + msg := fmt.Sprintf("create release image job error: %v", err) + p.Log.Error(msg) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = msg + return + } + p.Log.Infof("succeed to create artifact package job %s", p.JobName) +} + +// SetTask ... +func (p *ArtifactPackageTaskPlugin) SetTask(t map[string]interface{}) error { + task, err := ToArtifactTask(t) + if err != nil { + return err + } + p.Task = task + return nil +} + +// GetTask ... +func (p *ArtifactPackageTaskPlugin) GetTask() interface{} { + return p.Task +} + +// Wait ... +func (p *ArtifactPackageTaskPlugin) Wait(ctx context.Context) { + + status := p.waitJobStart() + if status != config.StatusRunning { + p.SetStatus(status) + return + } + + status = p.waitJobEnd(ctx, p.TaskTimeout()) + p.SetProgress(p.RealTimeProgress) + p.SetStatus(status) +} + +func (p *ArtifactPackageTaskPlugin) waitJobStart() config.Status { + namespace, jobName, kubeClient, xl := p.KubeNamespace, p.JobName, p.kubeClient, p.Log + xl.Infof("wait job to start: %s/%s", namespace, jobName) + podTimeout := time.After(120 * time.Second) + var started bool + for { + select { + case <-podTimeout: + return config.StatusTimeout + default: + job, _, err := getter.GetJob(namespace, jobName, kubeClient) + if err != nil { + xl.Errorf("get job failed, namespace:%s, jobName:%s, err:%v", namespace, jobName, err) + } + if job != nil { + started = job.Status.Active > 0 + } + } + if started { + break + } + + time.Sleep(time.Second) + } + return config.StatusRunning +} + +func (p *ArtifactPackageTaskPlugin) waitJobEnd(ctx context.Context, taskTimeout int) (status config.Status) { + namespace, jobName, kubeClient, xl := p.KubeNamespace, p.JobName, p.kubeClient, p.Log + xl.Infof("wait job to start: %s/%s", namespace, jobName) + timeout := time.After(time.Duration(taskTimeout) * time.Second) + + // 等待job 运行结束 + xl.Infof("wait job to end: %s %s", namespace, jobName) + for { + select { + case <-ctx.Done(): + return config.StatusCancelled + + case <-timeout: + return config.StatusTimeout + + default: + job, found, err := getter.GetJob(namespace, jobName, kubeClient) + if err != nil || !found { + xl.Errorf("failed to get pod with label job-name=%s %v", jobName, err) + return config.StatusFailed + } + // pod is still running + if job.Status.Active != 0 { + + pods, err := getter.ListPods(namespace, labels.Set{"job-name": jobName}.AsSelector(), kubeClient) + if err != nil { + xl.Errorf("failed to find pod with label job-name=%s %v", jobName, err) + return config.StatusFailed + } + + var done bool + for _, pod := range pods { + ipod := wrapper.Pod(pod) + if ipod.Pending() { + continue + } + if ipod.Failed() { + return config.StatusFailed + } + + if !ipod.Finished() { + progressInfo, err := getProgressInfo(namespace, ipod.Name, ipod.ContainerNames()[0], setting.ProgressFile) + if err != nil { + xl.Infof("failed to check dog food file %s %v", pods[0].Name, err) + break + } + p.RealTimeProgress = progressInfo + xl.Infof("find progress info %s", progressInfo) + p.SetProgress(progressInfo) + p.SetStatus(config.StatusRunning) + + if p.AckFunc != nil { + p.AckFunc() + } + } else { + done = true + } + } + + if done { + return config.StatusPassed + } + } else if job.Status.Succeeded != 0 { + return config.StatusPassed + } else { + return config.StatusFailed + } + } + + time.Sleep(time.Second * 5) + } + +} + +// Complete ... +func (p *ArtifactPackageTaskPlugin) Complete(ctx context.Context, pipelineTask *task.Task, serviceName string) { + jobLabel := &JobLabel{ + PipelineName: pipelineTask.PipelineName, + ServiceName: serviceName, + TaskID: pipelineTask.TaskID, + TaskType: string(p.Type()), + PipelineType: string(pipelineTask.Type), + } + + // 清理用户取消和超时的任务 + defer func() { + if p.Task.TaskStatus == config.StatusCancelled || p.Task.TaskStatus == config.StatusTimeout { + if err := ensureDeleteJob(p.KubeNamespace, jobLabel, p.kubeClient); err != nil { + p.Log.Error(err) + p.Task.Error = err.Error() + } + return + } + }() + + // 保存实时日志到s3 + err := saveContainerLog(pipelineTask, p.KubeNamespace, "", p.FileName, jobLabel, p.kubeClient) + if err != nil { + p.Log.Error(err) + p.Task.Error = err.Error() + return + } + + p.Task.LogFile = p.JobName +} + +// IsTaskDone ... +func (p *ArtifactPackageTaskPlugin) IsTaskDone() bool { + if p.Task.TaskStatus != config.StatusCreated && p.Task.TaskStatus != config.StatusRunning { + return true + } + return false +} + +// IsTaskFailed ... +func (p *ArtifactPackageTaskPlugin) IsTaskFailed() bool { + if p.Task.TaskStatus == config.StatusFailed || p.Task.TaskStatus == config.StatusTimeout || p.Task.TaskStatus == config.StatusCancelled { + return true + } + return false +} + +// SetStartTime ... +func (p *ArtifactPackageTaskPlugin) SetStartTime() { + p.Task.StartTime = time.Now().Unix() +} + +// SetEndTime ... +func (p *ArtifactPackageTaskPlugin) SetEndTime() { + p.Task.EndTime = time.Now().Unix() +} + +// IsTaskEnabled ... +func (p *ArtifactPackageTaskPlugin) IsTaskEnabled() bool { + return p.Task.Enabled +} + +// ResetError ... +func (p *ArtifactPackageTaskPlugin) ResetError() { + p.Task.Error = "" +} + +func getProgressInfo(namespace string, pod string, container string, progressInfoFile string) (string, error) { + stdOut, stdErr, success, err := podexec.ExecWithOptions(podexec.ExecOptions{ + Command: []string{"cat", progressInfoFile}, + Namespace: namespace, + PodName: pod, + ContainerName: container, + }) + + if err != nil { + return "", err + } + if len(stdErr) != 0 { + return "", fmt.Errorf(stdErr) + } + if !success { + return "", fmt.Errorf("pod exec execute failed") + } + return stdOut, nil +} diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/build.go b/pkg/microservice/warpdrive/core/service/taskplugin/build.go index e9f614f9a839a2c5a7a4e3d1b52e0d23c016c94d..c37dac2ea8fa13a00ebdfe0c8d120bea6bee2b0a 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/build.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/build.go @@ -19,28 +19,33 @@ package taskplugin import ( "context" "fmt" + "math" "math/rand" + "os" "strconv" "strings" "time" "go.uber.org/zap" "gopkg.in/yaml.v3" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" - configbase "github.com/koderover/zadig/pkg/config" + zadigconfig "github.com/koderover/zadig/pkg/config" "github.com/koderover/zadig/pkg/microservice/warpdrive/config" + "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/common" "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types/task" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/poetry" krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client" "github.com/koderover/zadig/pkg/tool/kube/updater" + "github.com/koderover/zadig/pkg/types" ) const ( - // BuildTaskV2Timeout ... - BuildTaskV2Timeout = 60 * 60 * 3 // 60 minutes + BuildTaskV2Timeout = 60 * 60 * 3 // 180 minutes ) // InitializeBuildTaskPlugin to initialize build task plugin, and return reference @@ -48,6 +53,8 @@ func InitializeBuildTaskPlugin(taskType config.TaskType) TaskPlugin { return &BuildTaskPlugin{ Name: taskType, kubeClient: krkubeclient.Client(), + clientset: krkubeclient.Clientset(), + restConfig: krkubeclient.RESTConfig(), } } @@ -58,6 +65,8 @@ type BuildTaskPlugin struct { JobName string FileName string kubeClient client.Client + clientset kubernetes.Interface + restConfig *rest.Config Task *task.Build Log *zap.SugaredLogger @@ -68,7 +77,6 @@ func (p *BuildTaskPlugin) SetAckFunc(ack func()) { p.ack = ack } -// Init ... func (p *BuildTaskPlugin) Init(jobname, filename string, xl *zap.SugaredLogger) { p.JobName = jobname p.Log = xl @@ -79,18 +87,19 @@ func (p *BuildTaskPlugin) Type() config.TaskType { return p.Name } -// Status ... func (p *BuildTaskPlugin) Status() config.Status { return p.Task.TaskStatus } -// SetStatus ... func (p *BuildTaskPlugin) SetStatus(status config.Status) { p.Task.TaskStatus = status } -// TaskTimeout ... +// Note: Unit of input `p.Task.Timeout` is `minutes` and convert it to `seconds` internally. +// TODO: Using time implicitly is easy to generate bugs. We need use `time.Duration` instead. func (p *BuildTaskPlugin) TaskTimeout() int { + p.Log.Infof("IsRestart: %t. TaskTimeout: %d", p.Task.IsRestart, p.Task.Timeout) + if p.Task.Timeout == 0 { p.Task.Timeout = BuildTaskV2Timeout } else { @@ -101,6 +110,13 @@ func (p *BuildTaskPlugin) TaskTimeout() int { return p.Task.Timeout } +// Note: This is a temporary function to be compatible with the `TaskTimeout()` method's process of time. +// TODO: Remove this function after `TaskTimeout` uses `time.Duration`. +func (p *BuildTaskPlugin) tmpSetTaskTimeout(durationInSeconds int) { + p.Task.Timeout = int(math.Ceil(float64(durationInSeconds) / 60.0)) + p.Task.IsRestart = false +} + func (p *BuildTaskPlugin) SetBuildStatusCompleted(status config.Status) { p.Task.BuildStatus.Status = status p.Task.BuildStatus.EndTime = time.Now().Unix() @@ -108,16 +124,82 @@ func (p *BuildTaskPlugin) SetBuildStatusCompleted(status config.Status) { //TODO: Binded Archive File logic func (p *BuildTaskPlugin) Run(ctx context.Context, pipelineTask *task.Task, pipelineCtx *task.PipelineCtx, serviceName string) { + if p.Task.CacheEnable && !pipelineTask.ConfigPayload.ResetCache { + pipelineCtx.CacheEnable = true + pipelineCtx.Cache = p.Task.Cache + pipelineCtx.CacheDirType = p.Task.CacheDirType + pipelineCtx.CacheUserDir = p.Task.CacheUserDir + } else { + pipelineCtx.CacheEnable = false + } + + // TODO: Since the namespace field has been used continuously since v1.10.0, the processing logic related to namespace needs to + // be deleted in v1.11.0. + switch p.Task.ClusterID { + case setting.LocalClusterID: + p.KubeNamespace = zadigconfig.Namespace() + default: + p.KubeNamespace = setting.AttachedClusterNamespace + + crClient, clientset, restConfig, err := GetK8sClients(pipelineTask.ConfigPayload.HubServerAddr, p.Task.ClusterID) + if err != nil { + p.Log.Error(err) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = err.Error() + p.SetBuildStatusCompleted(config.StatusFailed) + return + } + + p.kubeClient = crClient + p.clientset = clientset + p.restConfig = restConfig + } + + dockerhosts := common.NewDockerHosts(pipelineTask.ConfigPayload.HubServerAddr, p.Log) + pipelineTask.DockerHost = dockerhosts.GetBestHost(common.ClusterID(p.Task.ClusterID), serviceName) + + // not local cluster + var ( + replaceDindServer = "." + DindServer + dockerHost = "" + ) + + if p.Task.ClusterID != "" && p.Task.ClusterID != setting.LocalClusterID { + if strings.Contains(pipelineTask.DockerHost, pipelineTask.ConfigPayload.Build.KubeNamespace) { + // replace namespace only + dockerHost = strings.Replace(pipelineTask.DockerHost, pipelineTask.ConfigPayload.Build.KubeNamespace, KoderoverAgentNamespace, 1) + } else { + // add namespace + dockerHost = strings.Replace(pipelineTask.DockerHost, replaceDindServer, replaceDindServer+"."+KoderoverAgentNamespace, 1) + } + } else if p.Task.ClusterID == "" || p.Task.ClusterID == setting.LocalClusterID { + if !strings.Contains(pipelineTask.DockerHost, pipelineTask.ConfigPayload.Build.KubeNamespace) { + // add namespace + dockerHost = strings.Replace(pipelineTask.DockerHost, replaceDindServer, replaceDindServer+"."+pipelineTask.ConfigPayload.Build.KubeNamespace, 1) + } + } + pipelineCtx.DockerHost = dockerHost + if pipelineTask.Type == config.WorkflowType { envName := pipelineTask.WorkflowArgs.Namespace envNameVar := &task.KeyVal{Key: "ENV_NAME", Value: envName, IsCredential: false} p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, envNameVar) } else if pipelineTask.Type == config.ServiceType { + // Note: + // In cloud host scenarios, this type of task is performed when creating the environment. + // This type of task does not occur in other scenarios. + envName := pipelineTask.ServiceTaskArgs.Namespace envNameVar := &task.KeyVal{Key: "ENV_NAME", Value: envName, IsCredential: false} p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, envNameVar) } + // Note: When 'pipelinetask.type == config.ServiceType', it may be `nil`. + if pipelineTask.WorkflowArgs != nil { + p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, &task.KeyVal{Key: "WORKFLOW", Value: pipelineTask.WorkflowArgs.WorkflowName}) + p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, &task.KeyVal{Key: "PROJECT", Value: pipelineTask.WorkflowArgs.ProductTmplName}) + } + taskIDVar := &task.KeyVal{Key: "TASK_ID", Value: strconv.FormatInt(pipelineTask.TaskID, 10), IsCredential: false} p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, taskIDVar) @@ -135,25 +217,42 @@ func (p *BuildTaskPlugin) Run(ctx context.Context, pipelineTask *task.Task, pipe p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, envHostKeysVar) } - p.KubeNamespace = pipelineTask.ConfigPayload.Build.KubeNamespace - for _, repo := range p.Task.JobCtx.Builds { - repoName := strings.Replace(repo.RepoName, "-", "_", -1) - if len(repo.Branch) > 0 { - branchVar := &task.KeyVal{Key: fmt.Sprintf("%s_BRANCH", repoName), Value: repo.Branch, IsCredential: false} - p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, branchVar) - } + // env host names + for envName, names := range p.Task.EnvHostNames { + envHostKeysVar := &task.KeyVal{Key: envName + "_HOST_NAMEs", Value: strings.Join(names, ","), IsCredential: false} + p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, envHostKeysVar) + } - if len(repo.Tag) > 0 { - tagVar := &task.KeyVal{Key: fmt.Sprintf("%s_TAG", repoName), Value: repo.Tag, IsCredential: false} - p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, tagVar) + // ARTIFACT + if p.Task.JobCtx.FileArchiveCtx != nil { + var workspace = "/workspace" + if pipelineTask.ConfigPayload.ClassicBuild { + workspace = pipelineCtx.Workspace } + artifactKeysVar := &task.KeyVal{Key: "ARTIFACT", Value: fmt.Sprintf("%s/%s/%s", workspace, p.Task.JobCtx.FileArchiveCtx.FileLocation, p.Task.JobCtx.FileArchiveCtx.FileName), IsCredential: false} + p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, artifactKeysVar) + } - if repo.PR > 0 { - prVar := &task.KeyVal{Key: fmt.Sprintf("%s_PR", repoName), Value: strconv.Itoa(repo.PR), IsCredential: false} - p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, prVar) + //instantiates variables like ${_BRANCH} ${${REPO_index}_BRANCH} .. + p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, InstantiateBuildSysVariables(&p.Task.JobCtx)...) + + // Note: Currently, `SERVICE` in the environment variable represents a service module. + // Since variable rendering is required next, the `SERVICE_MODULE` environment variable is added to accurately + // characterize the service module. + // Do not use 'p.task. ServiceName' as the value because it is in the 'ServiceModule_ServiceName' format. + for _, env := range p.Task.JobCtx.EnvVars { + if env.Key == "SERVICE" { + p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, &task.KeyVal{Key: "SERVICE_MODULE", Value: env.Value}) + break } } + // Since we allow users to use custom environment variables, variable resolution is required. + if pipelineCtx.CacheEnable && pipelineCtx.Cache.MediumType == types.NFSMedium { + pipelineCtx.CacheUserDir = p.renderEnv(pipelineCtx.CacheUserDir) + pipelineCtx.Cache.NFSProperties.Subpath = p.renderEnv(pipelineCtx.Cache.NFSProperties.Subpath) + } + jobCtx := JobCtxBuilder{ JobName: p.JobName, PipelineCtx: pipelineCtx, @@ -162,11 +261,6 @@ func (p *BuildTaskPlugin) Run(ctx context.Context, pipelineTask *task.Task, pipe Installs: p.Task.InstallCtx, } - poetryClient := poetry.New(configbase.PoetryServiceAddress(), config.PoetryAPIRootKey()) - if fs, err := poetryClient.ListFeatures(); err == nil { - pipelineTask.Features = fs - } - if p.Task.BuildStatus == nil { p.Task.BuildStatus = &task.BuildStatus{} } @@ -193,34 +287,41 @@ func (p *BuildTaskPlugin) Run(ctx context.Context, pipelineTask *task.Task, pipe PipelineType: string(pipelineTask.Type), } - if err := ensureDeleteConfigMap(p.KubeNamespace, jobLabel, p.kubeClient); err != nil { - p.Log.Error(err) - p.Task.TaskStatus = config.StatusFailed - p.Task.Error = err.Error() - p.SetBuildStatusCompleted(config.StatusFailed) - return - } - - if err := createJobConfigMap( - p.KubeNamespace, p.JobName, jobLabel, string(jobCtxBytes), p.kubeClient); err != nil { - msg := fmt.Sprintf("createJobConfigMap error: %v", err) + jobObj, jobExist, err := checkJobExists(ctx, p.KubeNamespace, jobLabel, p.kubeClient) + if err != nil { + msg := fmt.Sprintf("failed to check whether Job exist for %s:%d: %s", pipelineTask.PipelineName, pipelineTask.TaskID, err) p.Log.Error(msg) p.Task.TaskStatus = config.StatusFailed p.Task.Error = msg p.SetBuildStatusCompleted(config.StatusFailed) return } - p.Log.Infof("succeed to create cm for build job %s", p.JobName) + if jobExist { + p.Log.Infof("Job %s:%d eixsts.", pipelineTask.PipelineName, pipelineTask.TaskID) + + p.JobName = jobObj.Name + + // If the code is executed at this point, it indicates that the `wd` instance that executed the Job has been restarted and the + // Job timeout period needs to be corrected. + // + // Rule of reseting timeout: `timeout - (now - start_time_of_job) + compensate_duration` + // For now, `compensate_duration` is 2min. + taskTimeout := p.TaskTimeout() + elaspedTime := time.Now().Unix() - jobObj.Status.StartTime.Time.Unix() + timeout := taskTimeout + 120 - int(elaspedTime) + p.Log.Infof("Timeout before normalization: %d seconds", timeout) + if timeout < 0 { + // That shouldn't happen in theory, but a protection is needed. + timeout = 0 + } - jobImage := fmt.Sprintf("%s-%s", pipelineTask.ConfigPayload.Release.ReaperImage, p.Task.BuildOS) - if p.Task.ImageFrom == setting.ImageFromCustom { - jobImage = p.Task.BuildOS + p.Log.Infof("Timeout after normalization: %d seconds", timeout) + p.tmpSetTaskTimeout(timeout) } - //Resource request default value is LOW - job, err := buildJob(p.Type(), jobImage, p.JobName, serviceName, p.Task.ResReq, pipelineCtx, pipelineTask, p.Task.Registries) + _, cmExist, err := checkConfigMapExists(ctx, p.KubeNamespace, jobLabel, p.kubeClient) if err != nil { - msg := fmt.Sprintf("create build job context error: %v", err) + msg := fmt.Sprintf("failed to check whether ConfigMap exist for %s:%d: %s", pipelineTask.PipelineName, pipelineTask.TaskID, err) p.Log.Error(msg) p.Task.TaskStatus = config.StatusFailed p.Task.Error = msg @@ -228,40 +329,66 @@ func (p *BuildTaskPlugin) Run(ctx context.Context, pipelineTask *task.Task, pipe return } - job.Namespace = p.KubeNamespace + if !cmExist { + p.Log.Infof("ConfigMap for Job %s:%d does not exist. Create.", pipelineTask.PipelineName, pipelineTask.TaskID) - if err := ensureDeleteJob(p.KubeNamespace, jobLabel, p.kubeClient); err != nil { - msg := fmt.Sprintf("delete build job error: %v", err) - p.Log.Error(msg) - p.Task.TaskStatus = config.StatusFailed - p.Task.Error = msg - p.SetBuildStatusCompleted(config.StatusFailed) - return + err = createJobConfigMap(p.KubeNamespace, p.JobName, jobLabel, string(jobCtxBytes), p.kubeClient) + if err != nil && !apierrors.IsAlreadyExists(err) { + msg := fmt.Sprintf("createJobConfigMap error: %v", err) + p.Log.Error(msg) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = msg + p.SetBuildStatusCompleted(config.StatusFailed) + return + } } + p.Log.Infof("succeed to create cm for build job %s", p.JobName) - // 将集成到KodeRover的私有镜像仓库的访问权限设置到namespace中 - if err := createOrUpdateRegistrySecrets(p.KubeNamespace, p.Task.Registries, p.kubeClient); err != nil { - msg := fmt.Sprintf("create secret error: %v", err) - p.Log.Error(msg) - p.Task.TaskStatus = config.StatusFailed - p.Task.Error = msg - p.SetBuildStatusCompleted(config.StatusFailed) - return - } - if err := updater.CreateJob(job, p.kubeClient); err != nil { - msg := fmt.Sprintf("create build job error: %v", err) - p.Log.Error(msg) - p.Task.TaskStatus = config.StatusFailed - p.Task.Error = msg - p.SetBuildStatusCompleted(config.StatusFailed) - return + if !jobExist { + p.Log.Infof("Job %s:%d does not exist. Create.", pipelineTask.PipelineName, pipelineTask.TaskID) + + jobImage := getReaperImage(pipelineTask.ConfigPayload.Release.ReaperImage, p.Task.BuildOS, p.Task.ImageFrom) + + //Resource request default value is LOW + job, err := buildJob(p.Type(), jobImage, p.JobName, serviceName, p.Task.ClusterID, pipelineTask.ConfigPayload.Build.KubeNamespace, p.Task.ResReq, p.Task.ResReqSpec, pipelineCtx, pipelineTask, p.Task.Registries) + if err != nil { + msg := fmt.Sprintf("create build job context error: %v", err) + p.Log.Error(msg) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = msg + p.SetBuildStatusCompleted(config.StatusFailed) + return + } + + job.Namespace = p.KubeNamespace + + // Set imagePullSecrets of the private image registry integrated with KodeRover into the namespace. + if err := createOrUpdateRegistrySecrets(p.KubeNamespace, pipelineTask.ConfigPayload.RegistryID, p.Task.Registries, p.kubeClient); err != nil { + msg := fmt.Sprintf("create secret error: %v", err) + p.Log.Error(msg) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = msg + p.SetBuildStatusCompleted(config.StatusFailed) + return + } + + err = updater.CreateJob(job, p.kubeClient) + if err != nil && !apierrors.IsAlreadyExists(err) { + msg := fmt.Sprintf("create build job error: %v", err) + p.Log.Error(msg) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = msg + p.SetBuildStatusCompleted(config.StatusFailed) + return + } } p.Log.Infof("succeed to create build job %s", p.JobName) + + p.Task.TaskStatus = waitJobReady(ctx, p.KubeNamespace, p.JobName, p.kubeClient, p.Log) } -// Wait ... func (p *BuildTaskPlugin) Wait(ctx context.Context) { - status := waitJobEndWithFile(ctx, p.TaskTimeout(), p.KubeNamespace, p.JobName, true, p.kubeClient, p.Log) + status := waitJobEndWithFile(ctx, p.TaskTimeout(), p.KubeNamespace, p.JobName, true, p.kubeClient, p.clientset, p.restConfig, p.Log) p.SetBuildStatusCompleted(status) if status == config.StatusPassed { @@ -289,7 +416,6 @@ func (p *BuildTaskPlugin) Wait(ctx context.Context) { p.SetStatus(status) } -// Complete ... func (p *BuildTaskPlugin) Complete(ctx context.Context, pipelineTask *task.Task, serviceName string) { jobLabel := &JobLabel{ PipelineName: pipelineTask.PipelineName, @@ -299,18 +425,20 @@ func (p *BuildTaskPlugin) Complete(ctx context.Context, pipelineTask *task.Task, PipelineType: string(pipelineTask.Type), } - // 清理用户取消和超时的任务 + // Clean up tasks that user canceled or timed out. defer func() { - if p.Task.TaskStatus == config.StatusCancelled || p.Task.TaskStatus == config.StatusTimeout { + go func() { if err := ensureDeleteJob(p.KubeNamespace, jobLabel, p.kubeClient); err != nil { p.Log.Error(err) - p.Task.Error = err.Error() } - return - } + + if err := ensureDeleteConfigMap(p.KubeNamespace, jobLabel, p.kubeClient); err != nil { + p.Log.Error(err) + } + }() }() - err := saveContainerLog(pipelineTask, p.KubeNamespace, p.FileName, jobLabel, p.kubeClient) + err := saveContainerLog(pipelineTask, p.KubeNamespace, p.Task.ClusterID, p.FileName, jobLabel, p.kubeClient) if err != nil { p.Log.Error(err) p.Task.Error = err.Error() @@ -320,22 +448,20 @@ func (p *BuildTaskPlugin) Complete(ctx context.Context, pipelineTask *task.Task, p.Task.LogFile = p.FileName } -// SetTask ... func (p *BuildTaskPlugin) SetTask(t map[string]interface{}) error { task, err := ToBuildTask(t) if err != nil { return err } p.Task = task + return nil } -// GetTask ... func (p *BuildTaskPlugin) GetTask() interface{} { return p.Task } -// IsTaskDone ... func (p *BuildTaskPlugin) IsTaskDone() bool { if p.Task.TaskStatus != config.StatusCreated && p.Task.TaskStatus != config.StatusRunning { return true @@ -343,7 +469,6 @@ func (p *BuildTaskPlugin) IsTaskDone() bool { return false } -// IsTaskFailed ... func (p *BuildTaskPlugin) IsTaskFailed() bool { if p.Task.TaskStatus == config.StatusFailed || p.Task.TaskStatus == config.StatusTimeout || p.Task.TaskStatus == config.StatusCancelled { return true @@ -351,22 +476,36 @@ func (p *BuildTaskPlugin) IsTaskFailed() bool { return false } -// SetStartTime ... func (p *BuildTaskPlugin) SetStartTime() { p.Task.StartTime = time.Now().Unix() } -// SetEndTime ... func (p *BuildTaskPlugin) SetEndTime() { p.Task.EndTime = time.Now().Unix() } -// IsTaskEnabled ... func (p *BuildTaskPlugin) IsTaskEnabled() bool { return p.Task.Enabled } -// ResetError ... func (p *BuildTaskPlugin) ResetError() { p.Task.Error = "" } + +// Note: Since there are few environment variables and few variables to be replaced, +// this method is temporarily used. +func (p *BuildTaskPlugin) renderEnv(data string) string { + mapper := func(data string) string { + for _, envar := range p.Task.JobCtx.EnvVars { + if data != envar.Key { + continue + } + + return envar.Value + } + + return fmt.Sprintf("$%s", data) + } + + return os.Expand(data, mapper) +} diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/build_v3.go b/pkg/microservice/warpdrive/core/service/taskplugin/build_v3.go new file mode 100644 index 0000000000000000000000000000000000000000..d3c3bb5cd8b4a80fc9a852b440dd900ec99a227e --- /dev/null +++ b/pkg/microservice/warpdrive/core/service/taskplugin/build_v3.go @@ -0,0 +1,327 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package taskplugin + +import ( + "context" + "fmt" + "math/rand" + "strconv" + "time" + + "go.uber.org/zap" + "gopkg.in/yaml.v3" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/koderover/zadig/pkg/microservice/warpdrive/config" + "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types/task" + krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client" + "github.com/koderover/zadig/pkg/tool/kube/updater" +) + +const ( + // BuildTaskV3Timeout ... + BuildTaskV3Timeout = 60 * 60 * 1 // 60 minutes +) + +// InitializeBuildTaskV3Plugin to initialize build task plugin, and return reference +func InitializeBuildTaskV3Plugin(taskType config.TaskType) TaskPlugin { + return &BuildTaskV3Plugin{ + Name: taskType, + kubeClient: krkubeclient.Client(), + clientset: krkubeclient.Clientset(), + restConfig: krkubeclient.RESTConfig(), + } +} + +// Note: Deprecated. +// BuildTaskV3Plugin is Plugin, name should be compatible with task type +type BuildTaskV3Plugin struct { + Name config.TaskType + KubeNamespace string + JobName string + FileName string + kubeClient client.Client + clientset kubernetes.Interface + restConfig *rest.Config + Task *task.Build + Log *zap.SugaredLogger + + ack func() +} + +func (p *BuildTaskV3Plugin) SetAckFunc(ack func()) { + p.ack = ack +} + +// Init ... +func (p *BuildTaskV3Plugin) Init(jobname, filename string, xl *zap.SugaredLogger) { + p.JobName = jobname + p.Log = xl + p.FileName = filename +} + +func (p *BuildTaskV3Plugin) Type() config.TaskType { + return p.Name +} + +// Status ... +func (p *BuildTaskV3Plugin) Status() config.Status { + return p.Task.TaskStatus +} + +// SetStatus ... +func (p *BuildTaskV3Plugin) SetStatus(status config.Status) { + p.Task.TaskStatus = status +} + +// TaskTimeout ... +func (p *BuildTaskV3Plugin) TaskTimeout() int { + if p.Task.Timeout == 0 { + p.Task.Timeout = BuildTaskV3Timeout + } else { + if !p.Task.IsRestart { + p.Task.Timeout = p.Task.Timeout * 60 + } + } + return p.Task.Timeout +} + +func (p *BuildTaskV3Plugin) SetBuildStatusCompleted(status config.Status) { + p.Task.BuildStatus.Status = status + p.Task.BuildStatus.EndTime = time.Now().Unix() +} + +func (p *BuildTaskV3Plugin) Run(ctx context.Context, pipelineTask *task.Task, pipelineCtx *task.PipelineCtx, serviceName string) { + taskIDVar := &task.KeyVal{Key: "TASK_ID", Value: strconv.FormatInt(pipelineTask.TaskID, 10), IsCredential: false} + p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, taskIDVar) + + p.KubeNamespace = pipelineTask.ConfigPayload.Build.KubeNamespace + + //instantiates variables like ${_BRANCH} ${${REPO_index}_BRANCH} ... + p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, InstantiateBuildSysVariables(&p.Task.JobCtx)...) + + jobCtx := JobCtxBuilder{ + JobName: p.JobName, + PipelineCtx: pipelineCtx, + ArchiveFile: p.Task.JobCtx.PackageFile, + JobCtx: p.Task.JobCtx, + Installs: p.Task.InstallCtx, + } + + if p.Task.BuildStatus == nil { + p.Task.BuildStatus = &task.BuildStatus{} + } + + p.Task.BuildStatus.Status = config.StatusRunning + p.Task.BuildStatus.StartTime = time.Now().Unix() + p.ack() + + jobCtxBytes, err := yaml.Marshal(jobCtx.BuildReaperContext(pipelineTask, serviceName)) + if err != nil { + msg := fmt.Sprintf("cannot reaper.Context data: %v", err) + p.Log.Error(msg) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = msg + p.SetBuildStatusCompleted(config.StatusFailed) + return + } + + jobLabel := &JobLabel{ + PipelineName: pipelineTask.PipelineName, + ServiceName: serviceName, + TaskID: pipelineTask.TaskID, + TaskType: string(p.Type()), + PipelineType: string(pipelineTask.Type), + } + + if err := ensureDeleteConfigMap(p.KubeNamespace, jobLabel, p.kubeClient); err != nil { + p.Log.Error(err) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = err.Error() + p.SetBuildStatusCompleted(config.StatusFailed) + return + } + + if err := createJobConfigMap( + p.KubeNamespace, p.JobName, jobLabel, string(jobCtxBytes), p.kubeClient); err != nil { + msg := fmt.Sprintf("createJobConfigMap error: %v", err) + p.Log.Error(msg) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = msg + p.SetBuildStatusCompleted(config.StatusFailed) + return + } + p.Log.Infof("succeed to create cm for build job %s", p.JobName) + + jobImage := getReaperImage(pipelineTask.ConfigPayload.Release.ReaperImage, p.Task.BuildOS, p.Task.ImageFrom) + + //Resource request default value is LOW + job, err := buildJob(p.Type(), jobImage, p.JobName, serviceName, "", pipelineTask.ConfigPayload.Build.KubeNamespace, p.Task.ResReq, p.Task.ResReqSpec, pipelineCtx, pipelineTask, p.Task.Registries) + if err != nil { + msg := fmt.Sprintf("create build job context error: %v", err) + p.Log.Error(msg) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = msg + p.SetBuildStatusCompleted(config.StatusFailed) + return + } + + job.Namespace = p.KubeNamespace + + if err := ensureDeleteJob(p.KubeNamespace, jobLabel, p.kubeClient); err != nil { + msg := fmt.Sprintf("delete build job error: %v", err) + p.Log.Error(msg) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = msg + p.SetBuildStatusCompleted(config.StatusFailed) + return + } + + // 将集成到KodeRover的私有镜像仓库的访问权限设置到namespace中 + if err := createOrUpdateRegistrySecrets(p.KubeNamespace, pipelineTask.ConfigPayload.RegistryID, p.Task.Registries, p.kubeClient); err != nil { + msg := fmt.Sprintf("create secret error: %v", err) + p.Log.Error(msg) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = msg + p.SetBuildStatusCompleted(config.StatusFailed) + return + } + if err := updater.CreateJob(job, p.kubeClient); err != nil { + msg := fmt.Sprintf("create build job error: %v", err) + p.Log.Error(msg) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = msg + p.SetBuildStatusCompleted(config.StatusFailed) + return + } + p.Log.Infof("succeed to create build job %s", p.JobName) +} + +// Wait ... +func (p *BuildTaskV3Plugin) Wait(ctx context.Context) { + status := waitJobEndWithFile(ctx, p.TaskTimeout(), p.KubeNamespace, p.JobName, true, p.kubeClient, p.clientset, p.restConfig, p.Log) + p.SetBuildStatusCompleted(status) + + if status == config.StatusPassed { + if p.Task.DockerBuildStatus == nil { + p.Task.DockerBuildStatus = &task.DockerBuildStatus{} + } + + p.Task.DockerBuildStatus.StartTime = time.Now().Unix() + p.Task.DockerBuildStatus.Status = config.StatusRunning + p.ack() + + select { + case <-ctx.Done(): + p.Task.DockerBuildStatus.EndTime = time.Now().Unix() + p.Task.DockerBuildStatus.Status = config.StatusCancelled + p.Task.TaskStatus = config.StatusCancelled + p.ack() + return + case <-time.After(time.Duration(rand.Int()%2) * time.Second): + p.Task.DockerBuildStatus.EndTime = time.Now().Unix() + p.Task.DockerBuildStatus.Status = config.StatusPassed + } + } + + p.SetStatus(status) +} + +// Complete ... +func (p *BuildTaskV3Plugin) Complete(ctx context.Context, pipelineTask *task.Task, serviceName string) { + jobLabel := &JobLabel{ + PipelineName: pipelineTask.PipelineName, + ServiceName: serviceName, + TaskID: pipelineTask.TaskID, + TaskType: string(p.Type()), + PipelineType: string(pipelineTask.Type), + } + + // 清理用户取消和超时的任务 + defer func() { + if p.Task.TaskStatus == config.StatusCancelled || p.Task.TaskStatus == config.StatusTimeout { + if err := ensureDeleteJob(p.KubeNamespace, jobLabel, p.kubeClient); err != nil { + p.Log.Error(err) + p.Task.Error = err.Error() + } + return + } + }() + + err := saveContainerLog(pipelineTask, p.KubeNamespace, "", p.FileName, jobLabel, p.kubeClient) + if err != nil { + p.Log.Error(err) + p.Task.Error = err.Error() + return + } + + p.Task.LogFile = p.FileName +} + +// SetTask ... +func (p *BuildTaskV3Plugin) SetTask(t map[string]interface{}) error { + task, err := ToBuildTask(t) + if err != nil { + return err + } + p.Task = task + return nil +} + +// GetTask ... +func (p *BuildTaskV3Plugin) GetTask() interface{} { + return p.Task +} + +// IsTaskDone ... +func (p *BuildTaskV3Plugin) IsTaskDone() bool { + if p.Task.TaskStatus != config.StatusCreated && p.Task.TaskStatus != config.StatusRunning { + return true + } + return false +} + +// IsTaskFailed ... +func (p *BuildTaskV3Plugin) IsTaskFailed() bool { + if p.Task.TaskStatus == config.StatusFailed || p.Task.TaskStatus == config.StatusTimeout || p.Task.TaskStatus == config.StatusCancelled { + return true + } + return false +} + +// SetStartTime ... +func (p *BuildTaskV3Plugin) SetStartTime() { + p.Task.StartTime = time.Now().Unix() +} + +// SetEndTime ... +func (p *BuildTaskV3Plugin) SetEndTime() { + p.Task.EndTime = time.Now().Unix() +} + +// IsTaskEnabled ... +func (p *BuildTaskV3Plugin) IsTaskEnabled() bool { + return p.Task.Enabled +} + +// ResetError ... +func (p *BuildTaskV3Plugin) ResetError() { + p.Task.Error = "" +} diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/deploy.go b/pkg/microservice/warpdrive/core/service/taskplugin/deploy.go index 8f42958d87d046d1343f1aadba8c8e53a610f992..d50f29302af9be48ea3a6ab18834d7ae1b9277d1 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/deploy.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/deploy.go @@ -19,38 +19,29 @@ package taskplugin import ( "context" "fmt" - "path/filepath" "regexp" "strings" "time" - helmclient "github.com/mittwald/go-helm-client" "github.com/pkg/errors" "go.uber.org/zap" appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/yaml" configbase "github.com/koderover/zadig/pkg/config" "github.com/koderover/zadig/pkg/microservice/warpdrive/config" - "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/taskplugin/s3" "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types" "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types/task" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" + "github.com/koderover/zadig/pkg/shared/kube/resource" "github.com/koderover/zadig/pkg/shared/kube/wrapper" - helmtool "github.com/koderover/zadig/pkg/tool/helmclient" "github.com/koderover/zadig/pkg/tool/httpclient" krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client" "github.com/koderover/zadig/pkg/tool/kube/getter" - "github.com/koderover/zadig/pkg/tool/kube/multicluster" "github.com/koderover/zadig/pkg/tool/kube/updater" - s3tool "github.com/koderover/zadig/pkg/tool/s3" - "github.com/koderover/zadig/pkg/util" - "github.com/koderover/zadig/pkg/util/converter" - "github.com/koderover/zadig/pkg/util/fs" - yamlutil "github.com/koderover/zadig/pkg/util/yaml" ) // InitializeDeployTaskPlugin to initiate deploy task plugin and return ref @@ -60,8 +51,6 @@ func InitializeDeployTaskPlugin(taskType config.TaskType) TaskPlugin { kubeClient: krkubeclient.Client(), restConfig: krkubeclient.RESTConfig(), httpClient: httpclient.New( - httpclient.SetAuthScheme(setting.RootAPIKey), - httpclient.SetAuthToken(config.PoetryAPIRootKey()), httpclient.SetHostURL(configbase.AslanServiceAddress()), ), } @@ -84,8 +73,6 @@ func (p *DeployTaskPlugin) SetAckFunc(func()) { } const ( - // DeployTimeout ... - DeployTimeout = 60 * 10 // 10 minutes imageUrlParseRegexString = `(?P.+/)?(?P[^:]+){1}(:)?(?P.+)?` ) @@ -118,7 +105,7 @@ func (p *DeployTaskPlugin) SetStatus(status config.Status) { // TaskTimeout ... func (p *DeployTaskPlugin) TaskTimeout() int { if p.Task.Timeout == 0 { - p.Task.Timeout = DeployTimeout + p.Task.Timeout = setting.DeployTimeout } else { if !p.Task.IsRestart { p.Task.Timeout = p.Task.Timeout * 60 @@ -133,31 +120,13 @@ type EnvArgs struct { ProductName string `json:"product_name"` } -type SelectorBuilder struct { - ProductName string - GroupName string - ServiceName string - ConfigBackup string - EnvName string +type ResourceComponentSet interface { + GetName() string + GetAnnotations() map[string]string + GetContainers() []*resource.ContainerImage + GetKind() string } -const ( - // ProductLabel ... - ProductLabel = "s-product" - // GroupLabel ... - GroupLabel = "s-group" - // ServiceLabel ... - ServiceLabel = "s-service" - // ConfigBackupLabel ... - ConfigBackupLabel = "config-backup" - // NamespaceLabel - EnvNameLabel = "s-env" - // EnvName ... - UpdateBy = "update-by" - UpdateByID = "update-by-id" - UpdateTime = "update-time" -) - func (p *DeployTaskPlugin) Run(ctx context.Context, pipelineTask *task.Task, _ *task.PipelineCtx, _ string) { var ( err error @@ -174,532 +143,172 @@ func (p *DeployTaskPlugin) Run(ctx context.Context, pipelineTask *task.Task, _ * }() if pipelineTask.ConfigPayload.DeployClusterID != "" { - p.restConfig, err = multicluster.GetRESTConfig(pipelineTask.ConfigPayload.HubServerAddr, pipelineTask.ConfigPayload.DeployClusterID) + p.restConfig, err = kubeclient.GetRESTConfig(pipelineTask.ConfigPayload.HubServerAddr, pipelineTask.ConfigPayload.DeployClusterID) if err != nil { err = errors.WithMessage(err, "can't get k8s rest config") return } - p.kubeClient, err = multicluster.GetKubeClient(pipelineTask.ConfigPayload.HubServerAddr, pipelineTask.ConfigPayload.DeployClusterID) + p.kubeClient, err = kubeclient.GetKubeClient(pipelineTask.ConfigPayload.HubServerAddr, pipelineTask.ConfigPayload.DeployClusterID) if err != nil { err = errors.WithMessage(err, "can't init k8s client") return } } + containerName := p.Task.ContainerName + containerName = strings.TrimSuffix(containerName, "_"+p.Task.ServiceName) - if p.Task.ServiceType != setting.HelmDeployType { - // get servcie info - var ( - serviceInfo *types.ServiceTmpl - selector labels.Selector - ) - serviceInfo, err = p.getService(ctx, p.Task.ServiceName, p.Task.ServiceType, p.Task.ProductName) + // get servcie info + var ( + serviceInfo *types.ServiceTmpl + selector labels.Selector + ) + serviceInfo, err = p.getService(ctx, p.Task.ServiceName, p.Task.ServiceType, p.Task.ProductName, 0) + if err != nil { + // Maybe it is a share service, the entity is not under the project + serviceInfo, err = p.getService(ctx, p.Task.ServiceName, p.Task.ServiceType, "", 0) if err != nil { return } - if serviceInfo.WorkloadType == "" { - selector := labels.Set{setting.ProductLabel: p.Task.ProductName, setting.ServiceLabel: p.Task.ServiceName}.AsSelector() - - var deployments []*appsv1.Deployment - deployments, err = getter.ListDeployments(p.Task.Namespace, selector, p.kubeClient) - if err != nil { - return - } - - var statefulSets []*appsv1.StatefulSet - statefulSets, err = getter.ListStatefulSets(p.Task.Namespace, selector, p.kubeClient) - if err != nil { - return - } + } + if serviceInfo.WorkloadType == "" { + selector = labels.Set{setting.ProductLabel: p.Task.ProductName, setting.ServiceLabel: p.Task.ServiceName}.AsSelector() - L: - for _, deploy := range deployments { - for _, container := range deploy.Spec.Template.Spec.Containers { - if container.Name == p.Task.ContainerName { - err = updater.UpdateDeploymentImage(deploy.Namespace, deploy.Name, p.Task.ContainerName, p.Task.Image, p.kubeClient) - if err != nil { - err = errors.WithMessagef( - err, - "failed to update container image in %s/deployments/%s/%s", - p.Task.Namespace, deploy.Name, container.Name) - return - } - p.Task.ReplaceResources = append(p.Task.ReplaceResources, task.Resource{ - Kind: setting.Deployment, - Container: container.Name, - Origin: container.Image, - Name: deploy.Name, - }) - replaced = true - break L - } - } - } - Loop: - for _, sts := range statefulSets { - for _, container := range sts.Spec.Template.Spec.Containers { - if container.Name == p.Task.ContainerName { - err = updater.UpdateStatefulSetImage(sts.Namespace, sts.Name, p.Task.ContainerName, p.Task.Image, p.kubeClient) - if err != nil { - err = errors.WithMessagef( - err, - "failed to update container image in %s/statefulsets/%s/%s", - p.Task.Namespace, sts.Name, container.Name) - return - } - p.Task.ReplaceResources = append(p.Task.ReplaceResources, task.Resource{ - Kind: setting.StatefulSet, - Container: container.Name, - Origin: container.Image, - Name: sts.Name, - }) - replaced = true - break Loop - } - } - } - } else { - switch serviceInfo.WorkloadType { - case setting.StatefulSet: - var statefulSet *appsv1.StatefulSet - statefulSet, _, err = getter.GetStatefulSet(p.Task.Namespace, p.Task.ServiceName, p.kubeClient) - if err != nil { - return - } - for _, container := range statefulSet.Spec.Template.Spec.Containers { - if container.Name == p.Task.ContainerName { - err = updater.UpdateStatefulSetImage(statefulSet.Namespace, statefulSet.Name, p.Task.ContainerName, p.Task.Image, p.kubeClient) - if err != nil { - err = errors.WithMessagef( - err, - "failed to update container image in %s/statefulsets/%s/%s", - p.Task.Namespace, statefulSet.Name, container.Name) - return - } - p.Task.ReplaceResources = append(p.Task.ReplaceResources, task.Resource{ - Kind: setting.StatefulSet, - Container: container.Name, - Origin: container.Image, - Name: statefulSet.Name, - }) - replaced = true - break - } - } - case setting.Deployment: - var deployment *appsv1.Deployment - deployment, _, err = getter.GetDeployment(p.Task.Namespace, p.Task.ServiceName, p.kubeClient) - if err != nil { - return - } - for _, container := range deployment.Spec.Template.Spec.Containers { - if container.Name == p.Task.ContainerName { - err = updater.UpdateDeploymentImage(deployment.Namespace, deployment.Name, p.Task.ContainerName, p.Task.Image, p.kubeClient) - if err != nil { - err = errors.WithMessagef( - err, - "failed to update container image in %s/deployments/%s/%s", - p.Task.Namespace, deployment.Name, container.Name) - return - } - p.Task.ReplaceResources = append(p.Task.ReplaceResources, task.Resource{ - Kind: setting.Deployment, - Container: container.Name, - Origin: container.Image, - Name: deployment.Name, - }) - replaced = true - break - } - } - } + var deployments []*appsv1.Deployment + deployments, err = getter.ListDeployments(p.Task.Namespace, selector, p.kubeClient) + if err != nil { + return } - if !replaced { - err = errors.Errorf( - "container %s is not found in resources with label %s", p.Task.ContainerName, selector) + + var statefulSets []*appsv1.StatefulSet + statefulSets, err = getter.ListStatefulSets(p.Task.Namespace, selector, p.kubeClient) + if err != nil { return } - } else if p.Task.ServiceType == setting.HelmDeployType { - var ( - productInfo *types.Product - renderChart *types.RenderChart - replacedValuesYaml string - mergedValuesYaml string - replacedMergedValuesYaml string - servicePath string - chartPath string - replaceValuesMap map[string]interface{} - renderInfo *types.RenderSet - helmClient helmclient.Client - ) - - deployments, _ := getter.ListDeployments(p.Task.Namespace, nil, p.kubeClient) + + L: for _, deploy := range deployments { for _, container := range deploy.Spec.Template.Spec.Containers { - if strings.Contains(container.Image, p.Task.ContainerName) { - p.Log.Infof("deployments find match container.name:%s", container.Name) + if container.Name == containerName { + err = updater.UpdateDeploymentImage(deploy.Namespace, deploy.Name, containerName, p.Task.Image, p.kubeClient) + if err != nil { + err = errors.WithMessagef( + err, + "failed to update container image in %s/deployments/%s/%s", + p.Task.Namespace, deploy.Name, container.Name) + return + } p.Task.ReplaceResources = append(p.Task.ReplaceResources, task.Resource{ Kind: setting.Deployment, Container: container.Name, Origin: container.Image, Name: deploy.Name, }) + replaced = true + break L } } } - - statefulSets, _ := getter.ListStatefulSets(p.Task.Namespace, nil, p.kubeClient) + Loop: for _, sts := range statefulSets { for _, container := range sts.Spec.Template.Spec.Containers { - if strings.Contains(container.Image, p.Task.ContainerName) { - p.Log.Infof("statefulSets find match container.name:%s", container.Name) + if container.Name == containerName { + err = updater.UpdateStatefulSetImage(sts.Namespace, sts.Name, containerName, p.Task.Image, p.kubeClient) + if err != nil { + err = errors.WithMessagef( + err, + "failed to update container image in %s/statefulsets/%s/%s", + p.Task.Namespace, sts.Name, container.Name) + return + } p.Task.ReplaceResources = append(p.Task.ReplaceResources, task.Resource{ Kind: setting.StatefulSet, Container: container.Name, Origin: container.Image, Name: sts.Name, }) + replaced = true + break Loop } } } - - p.Log.Infof("start helm deploy, productName %s serviceName %s containerName %s namespace %s", p.Task.ProductName, - p.Task.ServiceName, p.Task.ContainerName, p.Task.Namespace) - - productInfo, err = p.getProductInfo(ctx, &EnvArgs{EnvName: p.Task.EnvName, ProductName: p.Task.ProductName}) - if err != nil { - err = errors.WithMessagef( - err, - "failed to get product %s/%s", - p.Task.Namespace, p.Task.ServiceName) - return - } - renderInfo, err = p.getRenderSet(ctx, productInfo.Render.Name, productInfo.Render.Revision) - if err != nil { - err = errors.WithMessagef( - err, - "failed to get getRenderSet %s/%d", - productInfo.Render.Name, productInfo.Render.Revision) - return - } - - var targetContainer *types.Container - for _, serviceGroup := range productInfo.Services { - for _, service := range serviceGroup { - if service.ServiceName == p.Task.ServiceName { - for _, container := range service.Containers { - if container.Name == p.Task.ContainerName { - targetContainer = container - } + } else { + switch serviceInfo.WorkloadType { + case setting.StatefulSet: + var statefulSet *appsv1.StatefulSet + statefulSet, _, err = getter.GetStatefulSet(p.Task.Namespace, p.Task.ServiceName, p.kubeClient) + if err != nil { + return + } + for _, container := range statefulSet.Spec.Template.Spec.Containers { + if container.Name == containerName { + err = updater.UpdateStatefulSetImage(statefulSet.Namespace, statefulSet.Name, containerName, p.Task.Image, p.kubeClient) + if err != nil { + err = errors.WithMessagef( + err, + "failed to update container image in %s/statefulsets/%s/%s", + p.Task.Namespace, statefulSet.Name, container.Name) + return } + p.Task.ReplaceResources = append(p.Task.ReplaceResources, task.Resource{ + Kind: setting.StatefulSet, + Container: container.Name, + Origin: container.Image, + Name: statefulSet.Name, + }) + replaced = true + break } } - } - - if targetContainer == nil { - err = errors.Errorf("failed to find target container %s from service %s", p.Task.ContainerName, p.Task.ServiceName) - return - } - - for _, chartInfo := range renderInfo.ChartInfos { - if chartInfo.ServiceName == p.Task.ServiceName { - renderChart = chartInfo - break + case setting.Deployment: + var deployment *appsv1.Deployment + deployment, _, err = getter.GetDeployment(p.Task.Namespace, p.Task.ServiceName, p.kubeClient) + if err != nil { + return } - } - - if renderChart == nil { - err = errors.Errorf("failed to update container image in %s/%s，not find", - p.Task.Namespace, p.Task.ServiceName) - return - } - - serviceValuesYaml := renderChart.ValuesYaml - - // prepare image replace info - validMatchData := getValidMatchData(targetContainer.ImagePath) - - replaceValuesMap, err = assignImageData(p.Task.Image, validMatchData) - if err != nil { - err = errors.WithMessagef( - err, - "failed to pase image uri %s/%s", - p.Task.Namespace, p.Task.ServiceName) - return - } - - // replace image into service's values.yaml - replacedValuesYaml, err = replaceImage(serviceValuesYaml, replaceValuesMap) - if err != nil { - err = errors.WithMessagef( - err, - "failed to replace image uri %s/%s", - p.Task.Namespace, p.Task.ServiceName) - return - } - if replacedValuesYaml == "" { - err = errors.Errorf("failed to set new image uri into service's values.yaml %s/%s", - p.Task.Namespace, p.Task.ServiceName) - return - } - - // merge override values and kvs into service's yaml - mergedValuesYaml, err = helmtool.MergeOverrideValues(serviceValuesYaml, renderChart.GetOverrideYaml(), renderChart.OverrideValues) - if err != nil { - err = errors.WithMessagef( - err, - "failed to merge override values %s", - renderChart.OverrideValues, - ) - return - } - - // replace image into final merged values.yaml - replacedMergedValuesYaml, err = replaceImage(mergedValuesYaml, replaceValuesMap) - if err != nil { - err = errors.WithMessagef( - err, - "failed to replace image uri into helm values %s/%s", - p.Task.Namespace, p.Task.ServiceName) - return - } - if replacedMergedValuesYaml == "" { - err = errors.Errorf("failed to set image uri into mreged values.yaml in %s/%s", - p.Task.Namespace, p.Task.ServiceName) - return - } - - p.Log.Infof("final replaced merged values: \n%s", replacedMergedValuesYaml) - - helmClient, err = helmtool.NewClientFromRestConf(p.restConfig, p.Task.Namespace) - if err != nil { - err = errors.WithMessagef( - err, - "failed to create helm client %s/%s", - p.Task.Namespace, p.Task.ServiceName) - return - } - - servicePath, err = p.downloadService(pipelineTask.ProductName, p.Task.ServiceName, pipelineTask.StorageURI) - if err != nil { - err = errors.WithMessagef( - err, - "failed to download service %s/%s", - p.Task.Namespace, p.Task.ServiceName) - return - } - chartPath, err = fs.RelativeToCurrentPath(servicePath) - if err != nil { - err = errors.WithMessagef( - err, - "failed to get relative path %s", - servicePath, - ) - return - } - - chartSpec := helmclient.ChartSpec{ - ReleaseName: util.GeneHelmReleaseName(p.Task.Namespace, p.Task.ServiceName), - ChartName: chartPath, - Namespace: p.Task.Namespace, - ReuseValues: true, - Version: renderChart.ChartVersion, - ValuesYaml: replacedMergedValuesYaml, - SkipCRDs: false, - UpgradeCRDs: true, - Timeout: time.Second * DeployTimeout, - } - - if _, err = helmClient.InstallOrUpgradeChart(context.TODO(), &chartSpec); err != nil { - err = errors.WithMessagef( - err, - "failed to Install helm chart %s/%s", - p.Task.Namespace, p.Task.ServiceName) - return - } - - //替换环境变量中的chartInfos - for _, chartInfo := range renderInfo.ChartInfos { - if chartInfo.ServiceName == p.Task.ServiceName { - chartInfo.ValuesYaml = replacedValuesYaml - break + for _, container := range deployment.Spec.Template.Spec.Containers { + if container.Name == containerName { + err = updater.UpdateDeploymentImage(deployment.Namespace, deployment.Name, containerName, p.Task.Image, p.kubeClient) + if err != nil { + err = errors.WithMessagef( + err, + "failed to update container image in %s/deployments/%s/%s", + p.Task.Namespace, deployment.Name, container.Name) + return + } + p.Task.ReplaceResources = append(p.Task.ReplaceResources, task.Resource{ + Kind: setting.Deployment, + Container: container.Name, + Origin: container.Image, + Name: deployment.Name, + }) + replaced = true + break + } } } - - // TODO too dangerous to override entire renderset! - err = p.updateRenderSet(ctx, &types.RenderSet{ - Name: renderInfo.Name, - Revision: renderInfo.Revision, - ChartInfos: renderInfo.ChartInfos, - }) - if err != nil { - err = errors.WithMessagef( - err, - "failed to update renderset info %s/%s, renderset %s", - p.Task.Namespace, p.Task.ServiceName, renderInfo.Name) - } } -} - -func (p *DeployTaskPlugin) getProductInfo(ctx context.Context, args *EnvArgs) (*types.Product, error) { - url := fmt.Sprintf("/api/environment/environments/%s/productInfo", args.ProductName) - - prod := &types.Product{} - _, err := p.httpClient.Get(url, httpclient.SetResult(prod), httpclient.SetQueryParam("envName", args.EnvName)) - if err != nil { - return nil, err + if !replaced { + err = errors.Errorf( + "container %s is not found in resources with label %s", containerName, selector) + return } - return prod, nil } -func (p *DeployTaskPlugin) getService(ctx context.Context, name, serviceType, productName string) (*types.ServiceTmpl, error) { +func (p *DeployTaskPlugin) getService(ctx context.Context, name, serviceType, productName string, revision int64) (*types.ServiceTmpl, error) { url := fmt.Sprintf("/api/service/services/%s/%s", name, serviceType) s := &types.ServiceTmpl{} - _, err := p.httpClient.Get(url, httpclient.SetResult(s), httpclient.SetQueryParam("productName", productName)) + _, err := p.httpClient.Get(url, httpclient.SetResult(s), httpclient.SetQueryParams(map[string]string{ + "projectName": productName, + "revision": fmt.Sprintf("%d", revision), + })) if err != nil { return nil, err } return s, nil } -func (p *DeployTaskPlugin) downloadService(productName, serviceName, storageURI string) (string, error) { - logger := p.Log - - base := configbase.LocalServicePath(productName, serviceName) - s3Storage, err := s3.NewS3StorageFromEncryptedURI(storageURI) - if err != nil { - return "", err - } - - tarball := fmt.Sprintf("%s.tar.gz", serviceName) - tarFilePath := filepath.Join(base, tarball) - s3Storage.Subfolder = filepath.Join(s3Storage.Subfolder, configbase.ObjectStorageServicePath(productName, serviceName)) - forcedPathStyle := true - if s3Storage.Provider == setting.ProviderSourceAli { - forcedPathStyle = false - } - client, err := s3tool.NewClient(s3Storage.Endpoint, s3Storage.Ak, s3Storage.Sk, s3Storage.Insecure, forcedPathStyle) - if err != nil { - p.Log.Errorf("failed to create s3 client, err: %+v", err) - return "", err - } - if err = client.Download(s3Storage.Bucket, s3Storage.GetObjectPath(tarball), tarFilePath); err != nil { - logger.Errorf("Failed to download file from s3, err: %s", err) - return "", err - } - - return tarFilePath, nil -} - -func (p *DeployTaskPlugin) getRenderSet(ctx context.Context, name string, revision int64) (*types.RenderSet, error) { - url := fmt.Sprintf("/api/project/renders/render/%s/revision/%d", name, revision) - - rs := &types.RenderSet{} - _, err := p.httpClient.Get(url, httpclient.SetResult(rs)) - if err != nil { - return nil, err - } - return rs, nil -} - -func (p *DeployTaskPlugin) updateRenderSet(ctx context.Context, args *types.RenderSet) error { - url := "/api/project/renders" - - _, err := p.httpClient.Put(url, httpclient.SetBody(args)) - - return err -} - -func getValidMatchData(spec *types.ImagePathSpec) map[string]string { - ret := make(map[string]string) - if spec.Repo != "" { - ret[setting.PathSearchComponentRepo] = spec.Repo - } - if spec.Image != "" { - ret[setting.PathSearchComponentImage] = spec.Image - } - if spec.Tag != "" { - ret[setting.PathSearchComponentTag] = spec.Tag - } - return ret -} - -// parse image url to map: repo=>xxx/xx/xx image=>xx tag=>xxx -func resolveImageUrl(imageUrl string) map[string]string { - subMatchAll := imageParseRegex.FindStringSubmatch(imageUrl) - result := make(map[string]string) - exNames := imageParseRegex.SubexpNames() - for i, matchedStr := range subMatchAll { - if i != 0 && matchedStr != "" && matchedStr != ":" { - result[exNames[i]] = matchedStr - } - } - return result -} - -// replace image defines in yaml by new version -func replaceImage(sourceYaml string, imageValuesMap map[string]interface{}) (string, error) { - nestedMap, err := converter.Expand(imageValuesMap) - if err != nil { - return "", err - } - bs, err := yaml.Marshal(nestedMap) - if err != nil { - return "", err - } - mergedBs, err := yamlutil.Merge([][]byte{[]byte(sourceYaml), bs}) - if err != nil { - return "", err - } - return string(mergedBs), nil -} - -// assignImageData assign image url data into match data -// matchData: image=>absolute-path repo=>absolute-path tag=>absolute-path -// return: absolute-image-path=>image-value absolute-repo-path=>repo-value absolute-tag-path=>tag-value -func assignImageData(imageUrl string, matchData map[string]string) (map[string]interface{}, error) { - ret := make(map[string]interface{}) - // total image url assigned into one single value - if len(matchData) == 1 { - for _, v := range matchData { - ret[v] = imageUrl - } - return ret, nil - } - - resolvedImageUrl := resolveImageUrl(imageUrl) - - // image url assigned into repo/image+tag - if len(matchData) == 3 { - ret[matchData[setting.PathSearchComponentRepo]] = strings.TrimSuffix(resolvedImageUrl[setting.PathSearchComponentRepo], "/") - ret[matchData[setting.PathSearchComponentImage]] = resolvedImageUrl[setting.PathSearchComponentImage] - ret[matchData[setting.PathSearchComponentTag]] = resolvedImageUrl[setting.PathSearchComponentTag] - return ret, nil - } - - if len(matchData) == 2 { - // image url assigned into repo/image + tag - if tagPath, ok := matchData[setting.PathSearchComponentTag]; ok { - ret[tagPath] = resolvedImageUrl[setting.PathSearchComponentTag] - for k, imagePath := range matchData { - if k == setting.PathSearchComponentTag { - continue - } - ret[imagePath] = fmt.Sprintf("%s%s", resolvedImageUrl[setting.PathSearchComponentRepo], resolvedImageUrl[setting.PathSearchComponentImage]) - break - } - return ret, nil - } - // image url assigned into repo + image(tag) - ret[matchData[setting.PathSearchComponentRepo]] = strings.TrimSuffix(resolvedImageUrl[setting.PathSearchComponentRepo], "/") - ret[matchData[setting.PathSearchComponentImage]] = fmt.Sprintf("%s:%s", resolvedImageUrl[setting.PathSearchComponentImage], resolvedImageUrl[setting.PathSearchComponentTag]) - return ret, nil - } - - return nil, errors.Errorf("match data illegal, expect length: 1-3, actual length: %d", len(matchData)) -} - // Wait ... func (p *DeployTaskPlugin) Wait(ctx context.Context) { // skip waiting for reset image task diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/deploy_helm.go b/pkg/microservice/warpdrive/core/service/taskplugin/deploy_helm.go new file mode 100644 index 0000000000000000000000000000000000000000..4713719bca2a327e71282a79b66e3d016348d970 --- /dev/null +++ b/pkg/microservice/warpdrive/core/service/taskplugin/deploy_helm.go @@ -0,0 +1,643 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package taskplugin + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "time" + + helmclient "github.com/mittwald/go-helm-client" + "github.com/pkg/errors" + "go.uber.org/zap" + "gopkg.in/yaml.v3" + "helm.sh/helm/v3/pkg/releaseutil" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + configbase "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/warpdrive/config" + "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/taskplugin/s3" + "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types" + "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types/task" + "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" + helmtool "github.com/koderover/zadig/pkg/tool/helmclient" + "github.com/koderover/zadig/pkg/tool/httpclient" + krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client" + s3tool "github.com/koderover/zadig/pkg/tool/s3" + "github.com/koderover/zadig/pkg/util/converter" + fsutil "github.com/koderover/zadig/pkg/util/fs" + yamlutil "github.com/koderover/zadig/pkg/util/yaml" +) + +// InitializeHelmDeployTaskPlugin initiates a plugin to deploy helm charts and return ref +func InitializeHelmDeployTaskPlugin(taskType config.TaskType) *HelmDeployTaskPlugin { + return &HelmDeployTaskPlugin{ + Name: taskType, + kubeClient: krkubeclient.Client(), + restConfig: krkubeclient.RESTConfig(), + httpClient: httpclient.New( + httpclient.SetHostURL(configbase.AslanServiceAddress()), + ), + } +} + +// HelmDeployTaskPlugin Plugin name should be compatible with task type +type HelmDeployTaskPlugin struct { + Name config.TaskType + JobName string + kubeClient client.Client + restConfig *rest.Config + Task *task.Deploy + Log *zap.SugaredLogger + ReplaceImage string + + ContentPlugins []*HelmDeployTaskPlugin + httpClient *httpclient.Client +} + +func (p *HelmDeployTaskPlugin) SetAckFunc(func()) {} + +func (p *HelmDeployTaskPlugin) Init(jobname, filename string, xl *zap.SugaredLogger) { + p.JobName = jobname + p.Log = xl +} + +func (p *HelmDeployTaskPlugin) Type() config.TaskType { + return p.Name +} + +func (p *HelmDeployTaskPlugin) Status() config.Status { + return p.Task.TaskStatus +} + +func (p *HelmDeployTaskPlugin) SetStatus(status config.Status) { + p.Task.TaskStatus = status + for _, subPlugin := range p.ContentPlugins { + subPlugin.Task.TaskStatus = status + } +} + +func (p *HelmDeployTaskPlugin) TaskTimeout() int { + if p.Task.Timeout == 0 { + p.Task.Timeout = setting.DeployTimeout + } else { + if !p.Task.IsRestart { + p.Task.Timeout = p.Task.Timeout * 60 + } + } + return p.Task.Timeout +} + +func (p *HelmDeployTaskPlugin) Run(ctx context.Context, pipelineTask *task.Task, _ *task.PipelineCtx, _ string) { + var ( + err error + ) + + defer func() { + if err != nil { + p.Log.Error(err) + p.SetStatus(config.StatusFailed) + p.Task.Error = err.Error() + return + } + }() + + if pipelineTask.ConfigPayload.DeployClusterID != "" { + p.restConfig, err = kubeclient.GetRESTConfig(pipelineTask.ConfigPayload.HubServerAddr, pipelineTask.ConfigPayload.DeployClusterID) + if err != nil { + err = errors.WithMessage(err, "can't get k8s rest config") + return + } + + p.kubeClient, err = kubeclient.GetKubeClient(pipelineTask.ConfigPayload.HubServerAddr, pipelineTask.ConfigPayload.DeployClusterID) + if err != nil { + err = errors.WithMessage(err, "can't init k8s client") + return + } + } + // all involved containers + containerNameSet := sets.NewString() + for _, sPlugin := range p.ContentPlugins { + singleContainerName := strings.TrimSuffix(sPlugin.Task.ContainerName, "_"+p.Task.ServiceName) + containerNameSet.Insert(singleContainerName) + } + + var ( + productInfo *types.Product + renderChart *types.RenderChart + replacedValuesYaml string + mergedValuesYaml string + replacedMergedValuesYaml string + servicePath string + chartPath string + replaceValuesMap map[string]interface{} + renderInfo *types.RenderSet + helmClient helmclient.Client + ) + + p.Log.Infof("start helm deploy, productName %s serviceName %s containerName %v envName: %s namespace %s", p.Task.ProductName, + p.Task.ServiceName, containerNameSet.List(), p.Task.EnvName, p.Task.Namespace) + + productInfo, err = p.getProductInfo(ctx, &EnvArgs{EnvName: p.Task.EnvName, ProductName: p.Task.ProductName}) + if err != nil { + err = errors.WithMessagef( + err, + "failed to get product %s/%s", + p.Task.Namespace, p.Task.ServiceName) + return + } + + renderInfo, err = p.getRenderSet(ctx, productInfo.Render.Name, productInfo.Render.Revision) + if err != nil { + err = errors.WithMessagef( + err, + "failed to get getRenderSet %s/%d", + productInfo.Render.Name, productInfo.Render.Revision) + return + } + + serviceRevisionInProduct := int64(0) + involvedImagePaths := make(map[string]*types.ImagePathSpec) + for _, service := range productInfo.GetServiceMap() { + if service.ServiceName != p.Task.ServiceName { + continue + } + serviceRevisionInProduct = service.Revision + for _, container := range service.Containers { + if !containerNameSet.Has(container.Name) { + continue + } + if container.ImagePath == nil { + err = errors.WithMessagef(err, "image path of %s/%s is nil", service.ServiceName, container.Name) + return + } + involvedImagePaths[container.Name] = container.ImagePath + } + break + } + + if len(involvedImagePaths) == 0 { + err = errors.Errorf("failed to find containers from service %s", p.Task.ServiceName) + return + } + + for _, chartInfo := range renderInfo.ChartInfos { + if chartInfo.ServiceName == p.Task.ServiceName { + renderChart = chartInfo + break + } + } + + if renderChart == nil { + err = errors.Errorf("failed to update container image in %s/%s，chart not found", + p.Task.Namespace, p.Task.ServiceName) + return + } + + // use revision of service currently applied in environment instead of the latest revision + path, errDownload := p.downloadService(pipelineTask.ProductName, p.Task.ServiceName, pipelineTask.StorageURI, serviceRevisionInProduct) + if errDownload != nil { + p.Log.Warnf("failed to get chart of revision: %d for service: %s, use latest version", + serviceRevisionInProduct, p.Task.ServiceName) + path, errDownload = p.downloadService(pipelineTask.ProductName, p.Task.ServiceName, + pipelineTask.StorageURI, 0) + if errDownload != nil { + err = errors.WithMessagef( + errDownload, + "failed to download service %s/%s", + p.Task.Namespace, p.Task.ServiceName) + return + } + } + + chartPath, err = fsutil.RelativeToCurrentPath(path) + if err != nil { + err = errors.WithMessagef( + err, + "failed to get relative path %s", + servicePath, + ) + return + } + + serviceValuesYaml := renderChart.ValuesYaml + replaceValuesMap = make(map[string]interface{}) + + for _, sPlugin := range p.ContentPlugins { + containerName := strings.TrimSuffix(sPlugin.Task.ContainerName, "_"+p.Task.ServiceName) + if imagePath, ok := involvedImagePaths[containerName]; ok { + validMatchData := getValidMatchData(imagePath) + singleReplaceValuesMap, errAssign := assignImageData(sPlugin.Task.Image, validMatchData) + if errAssign != nil { + err = errors.WithMessagef( + errAssign, + "failed to pase image uri %s/%s", + p.Task.Namespace, p.Task.ServiceName) + return + } + for k, v := range singleReplaceValuesMap { + replaceValuesMap[k] = v + } + } + } + + // replace image into service's values.yaml + replacedValuesYaml, err = replaceImage(serviceValuesYaml, replaceValuesMap) + if err != nil { + err = errors.WithMessagef( + err, + "failed to replace image uri %s/%s", + p.Task.Namespace, p.Task.ServiceName) + return + } + if replacedValuesYaml == "" { + err = errors.Errorf("failed to set new image uri into service's values.yaml %s/%s", + p.Task.Namespace, p.Task.ServiceName) + return + } + + // merge override values and kvs into service's yaml + mergedValuesYaml, err = helmtool.MergeOverrideValues(serviceValuesYaml, renderInfo.DefaultValues, renderChart.GetOverrideYaml(), renderChart.OverrideValues) + if err != nil { + err = errors.WithMessagef( + err, + "failed to merge override values %s", + renderChart.OverrideValues, + ) + return + } + + // replace image into final merged values.yaml + replacedMergedValuesYaml, err = replaceImage(mergedValuesYaml, replaceValuesMap) + if err != nil { + err = errors.WithMessagef( + err, + "failed to replace image uri into helm values %s/%s", + p.Task.Namespace, p.Task.ServiceName) + return + } + if replacedMergedValuesYaml == "" { + err = errors.Errorf("failed to set image uri into mreged values.yaml in %s/%s", + p.Task.Namespace, p.Task.ServiceName) + return + } + + p.Log.Infof("final replaced merged values: \n%s", replacedMergedValuesYaml) + + helmClient, err = helmtool.NewClientFromNamespace(pipelineTask.ConfigPayload.DeployClusterID, p.Task.Namespace) + if err != nil { + err = errors.WithMessagef( + err, + "failed to create helm client %s/%s", + p.Task.Namespace, p.Task.ServiceName) + return + } + + releaseName := p.Task.ReleaseName + + ensureUpgrade := func() error { + hrs, errHistory := helmClient.ListReleaseHistory(releaseName, 10) + if errHistory != nil { + // list history should not block deploy operation, error will be logged instead of returned + p.Log.Errorf("failed to list release history, release: %s, err: %s", releaseName, errHistory) + return nil + } + if len(hrs) == 0 { + return nil + } + releaseutil.Reverse(hrs, releaseutil.SortByRevision) + rel := hrs[0] + + if rel.Info.Status.IsPending() { + return fmt.Errorf("failed to upgrade release: %s with exceptional status: %s", releaseName, rel.Info.Status) + } + return nil + } + + err = ensureUpgrade() + if err != nil { + return + } + + timeOut := p.TaskTimeout() + chartSpec := helmclient.ChartSpec{ + ReleaseName: releaseName, + ChartName: chartPath, + Namespace: p.Task.Namespace, + ReuseValues: true, + Version: renderChart.ChartVersion, + ValuesYaml: replacedMergedValuesYaml, + SkipCRDs: false, + UpgradeCRDs: true, + Timeout: time.Second * time.Duration(timeOut), + Wait: true, + Replace: true, + MaxHistory: 10, + } + + done := make(chan bool) + go func(chan bool) { + if _, err = helmClient.InstallOrUpgradeChart(ctx, &chartSpec); err != nil { + err = errors.WithMessagef( + err, + "failed to upgrade helm chart %s/%s", + p.Task.Namespace, p.Task.ServiceName) + done <- false + } else { + done <- true + } + }(done) + + select { + case <-done: + break + case <-time.After(chartSpec.Timeout + time.Minute): + err = fmt.Errorf("failed to upgrade relase: %s, timeout", chartSpec.ReleaseName) + } + if err != nil { + return + } + + //替换环境变量中的chartInfos + for _, chartInfo := range renderInfo.ChartInfos { + if chartInfo.ServiceName == p.Task.ServiceName { + chartInfo.ValuesYaml = replacedValuesYaml + break + } + } + + // TODO too dangerous to override entire renderset! + err = p.updateRenderSet(ctx, &types.RenderSet{ + Name: renderInfo.Name, + Revision: renderInfo.Revision, + DefaultValues: renderInfo.DefaultValues, + ChartInfos: renderInfo.ChartInfos, + }) + if err != nil { + err = errors.WithMessagef( + err, + "failed to update renderset info %s/%s, renderset %s", + p.Task.Namespace, p.Task.ServiceName, renderInfo.Name) + } +} + +func (p *HelmDeployTaskPlugin) getProductInfo(ctx context.Context, args *EnvArgs) (*types.Product, error) { + url := fmt.Sprintf("/api/environment/environments/%s/productInfo", args.EnvName) + prod := &types.Product{} + _, err := p.httpClient.Get(url, httpclient.SetResult(prod), httpclient.SetQueryParam("projectName", args.ProductName), httpclient.SetQueryParam("ifPassFilter", "true")) + if err != nil { + return nil, err + } + return prod, nil +} + +// download chart info of specific version, use the latest version if fails +func (p *HelmDeployTaskPlugin) downloadService(productName, serviceName, storageURI string, revision int64) (string, error) { + logger := p.Log + + fileName := serviceName + if revision > 0 { + fileName = fmt.Sprintf("%s-%d", serviceName, revision) + } + tarball := fmt.Sprintf("%s.tar.gz", fileName) + localBase := configbase.LocalServicePath(productName, serviceName) + tarFilePath := filepath.Join(localBase, tarball) + + exists, err := fsutil.FileExists(tarFilePath) + if err != nil { + return "", err + } + if exists { + return tarFilePath, nil + } + + s3Storage, err := s3.NewS3StorageFromEncryptedURI(storageURI) + if err != nil { + return "", err + } + + s3Storage.Subfolder = filepath.Join(s3Storage.Subfolder, configbase.ObjectStorageServicePath(productName, serviceName)) + forcedPathStyle := true + if s3Storage.Provider == setting.ProviderSourceAli { + forcedPathStyle = false + } + s3Client, err := s3tool.NewClient(s3Storage.Endpoint, s3Storage.Ak, s3Storage.Sk, s3Storage.Insecure, forcedPathStyle) + if err != nil { + p.Log.Errorf("failed to create s3 client, err: %s", err) + return "", err + } + if err = s3Client.Download(s3Storage.Bucket, s3Storage.GetObjectPath(tarball), tarFilePath); err != nil { + logger.Errorf("failed to download file from s3, err: %s", err) + return "", err + } + + exists, err = fsutil.FileExists(tarFilePath) + if err != nil { + return "", err + } + if !exists { + return "", fmt.Errorf("file %s on s3 not found", s3Storage.GetObjectPath(tarball)) + } + return tarFilePath, nil +} + +func (p *HelmDeployTaskPlugin) getRenderSet(ctx context.Context, name string, revision int64) (*types.RenderSet, error) { + url := fmt.Sprintf("/api/project/renders/render/%s/revision/%d", name, revision) + rs := &types.RenderSet{} + _, err := p.httpClient.Get(url, httpclient.SetResult(rs), httpclient.SetQueryParam("ifPassFilter", "true")) + if err != nil { + return nil, err + } + return rs, nil +} + +func getValidMatchData(spec *types.ImagePathSpec) map[string]string { + ret := make(map[string]string) + if spec.Repo != "" { + ret[setting.PathSearchComponentRepo] = spec.Repo + } + if spec.Image != "" { + ret[setting.PathSearchComponentImage] = spec.Image + } + if spec.Tag != "" { + ret[setting.PathSearchComponentTag] = spec.Tag + } + return ret +} + +// replace image defines in yaml by new version +func replaceImage(sourceYaml string, imageValuesMap map[string]interface{}) (string, error) { + nestedMap, err := converter.Expand(imageValuesMap) + if err != nil { + return "", err + } + bs, err := yaml.Marshal(nestedMap) + if err != nil { + return "", err + } + mergedBs, err := yamlutil.Merge([][]byte{[]byte(sourceYaml), bs}) + if err != nil { + return "", err + } + return string(mergedBs), nil +} + +// parse image url to map: repo=>xxx/xx/xx image=>xx tag=>xxx +func resolveImageUrl(imageUrl string) map[string]string { + subMatchAll := imageParseRegex.FindStringSubmatch(imageUrl) + result := make(map[string]string) + exNames := imageParseRegex.SubexpNames() + for i, matchedStr := range subMatchAll { + if i != 0 && matchedStr != "" && matchedStr != ":" { + result[exNames[i]] = matchedStr + } + } + return result +} + +// assignImageData assign image url data into match data +// matchData: image=>absolute-path repo=>absolute-path tag=>absolute-path +// return: absolute-image-path=>image-value absolute-repo-path=>repo-value absolute-tag-path=>tag-value +func assignImageData(imageUrl string, matchData map[string]string) (map[string]interface{}, error) { + ret := make(map[string]interface{}) + // total image url assigned into one single value + if len(matchData) == 1 { + for _, v := range matchData { + ret[v] = imageUrl + } + return ret, nil + } + + resolvedImageUrl := resolveImageUrl(imageUrl) + + // image url assigned into repo/image+tag + if len(matchData) == 3 { + ret[matchData[setting.PathSearchComponentRepo]] = strings.TrimSuffix(resolvedImageUrl[setting.PathSearchComponentRepo], "/") + ret[matchData[setting.PathSearchComponentImage]] = resolvedImageUrl[setting.PathSearchComponentImage] + ret[matchData[setting.PathSearchComponentTag]] = resolvedImageUrl[setting.PathSearchComponentTag] + return ret, nil + } + + if len(matchData) == 2 { + // image url assigned into repo/image + tag + if tagPath, ok := matchData[setting.PathSearchComponentTag]; ok { + ret[tagPath] = resolvedImageUrl[setting.PathSearchComponentTag] + for k, imagePath := range matchData { + if k == setting.PathSearchComponentTag { + continue + } + ret[imagePath] = fmt.Sprintf("%s%s", resolvedImageUrl[setting.PathSearchComponentRepo], resolvedImageUrl[setting.PathSearchComponentImage]) + break + } + return ret, nil + } + // image url assigned into repo + image(tag) + ret[matchData[setting.PathSearchComponentRepo]] = strings.TrimSuffix(resolvedImageUrl[setting.PathSearchComponentRepo], "/") + ret[matchData[setting.PathSearchComponentImage]] = fmt.Sprintf("%s:%s", resolvedImageUrl[setting.PathSearchComponentImage], resolvedImageUrl[setting.PathSearchComponentTag]) + return ret, nil + } + + return nil, errors.Errorf("match data illegal, expect length: 1-3, actual length: %d", len(matchData)) +} + +func (p *HelmDeployTaskPlugin) updateRenderSet(ctx context.Context, args *types.RenderSet) error { + url := "/api/project/renders" + _, err := p.httpClient.Put(url, httpclient.SetBody(args), httpclient.SetQueryParam("ifPassFilter", "true")) + return err +} + +// Wait ... +func (p *HelmDeployTaskPlugin) Wait(ctx context.Context) { + // skip waiting for reset image task + if p.Task.SkipWaiting { + p.SetStatus(config.StatusPassed) + return + } + + // for services deployed by helm, use --wait option to ensure related resources are updated + p.SetStatus(config.StatusPassed) + return +} + +func (p *HelmDeployTaskPlugin) Complete(ctx context.Context, pipelineTask *task.Task, serviceName string) { +} + +func (p *HelmDeployTaskPlugin) SetTask(t map[string]interface{}) error { + if p.Task == nil { + task, err := ToDeployTask(t) + if err != nil { + return err + } + p.Task = task + } + return nil +} + +// GetTask ... +func (p *HelmDeployTaskPlugin) GetTask() interface{} { + return p.Task +} + +// IsTaskDone ... +func (p *HelmDeployTaskPlugin) IsTaskDone() bool { + if p.Task.TaskStatus != config.StatusCreated && p.Task.TaskStatus != config.StatusRunning { + return true + } + return false +} + +// IsTaskFailed ... +func (p *HelmDeployTaskPlugin) IsTaskFailed() bool { + if p.Task.TaskStatus == config.StatusFailed || p.Task.TaskStatus == config.StatusTimeout || p.Task.TaskStatus == config.StatusCancelled { + return true + } + return false +} + +// SetStartTime ... +func (p *HelmDeployTaskPlugin) SetStartTime() { + p.Task.StartTime = time.Now().Unix() + for _, subPlugin := range p.ContentPlugins { + subPlugin.Task.StartTime = p.Task.StartTime + } +} + +// SetEndTime ... +func (p *HelmDeployTaskPlugin) SetEndTime() { + p.Task.EndTime = time.Now().Unix() + for _, subPlugin := range p.ContentPlugins { + subPlugin.Task.EndTime = p.Task.EndTime + } +} + +// IsTaskEnabled ... +func (p *HelmDeployTaskPlugin) IsTaskEnabled() bool { + return p.Task.Enabled +} + +// ResetError ... +func (p *HelmDeployTaskPlugin) ResetError() { + p.Task.Error = "" + for _, subPlugin := range p.ContentPlugins { + subPlugin.Task.Error = "" + } +} diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/deploy_test.go b/pkg/microservice/warpdrive/core/service/taskplugin/deploy_test.go index 9cb71c12c67b44b98c682a9c964d2d5828b2de98..ddcca068544b80974ee585e2f5bbb5adba6829ae 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/deploy_test.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/deploy_test.go @@ -33,12 +33,12 @@ env: dev ingressClassName: koderover-admin-nginx image: - - repository: ccr.ccs.tencentyun.com/trial/go-sample-site + - repository: koderover.tencentcloudcr.com/test/go-sample-site pullPolicy: IfNotPresent tag: "0.1.0" imageNormal: - repository: ccr.ccs.tencentyun.com/trial/go-sample-site + repository: koderover.tencentcloudcr.com/test/go-sample-site pullPolicy: IfNotPresent tag: "0.1.0" diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/docker_build.go b/pkg/microservice/warpdrive/core/service/taskplugin/docker_build.go index b963e3937556eef1dbd4e452229cb1e25419d57c..350c95780dc98fd31d83d8382c42ebd28824d017 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/docker_build.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/docker_build.go @@ -24,6 +24,8 @@ import ( "go.uber.org/zap" yaml "gopkg.in/yaml.v3" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/koderover/zadig/pkg/microservice/warpdrive/config" @@ -41,6 +43,8 @@ type DockerBuildPlugin struct { JobName string FileName string kubeClient client.Client + clientset kubernetes.Interface + restConfig *rest.Config Task *task.DockerBuild Log *zap.SugaredLogger } @@ -53,11 +57,14 @@ const ( DockerBuildTimeout = 60 * 60 // 60 minutes ) +// Note: Deprecated. // InitializeDockerBuildTaskPlugin init docker build plugin func InitializeDockerBuildTaskPlugin(taskType config.TaskType) TaskPlugin { return &DockerBuildPlugin{ Name: taskType, kubeClient: krkubeclient.Client(), + clientset: krkubeclient.Clientset(), + restConfig: krkubeclient.RESTConfig(), } } @@ -134,6 +141,7 @@ func (p *DockerBuildPlugin) Run(ctx context.Context, pipelineTask *task.Task, pi JobType: setting.BuildImageJob, //Docker build context DockerBuildCtx: &task.DockerBuildCtx{ + Source: p.Task.Source, WorkDir: fmt.Sprintf("%s/%s", pipelineCtx.Workspace, p.Task.WorkDir), DockerFile: fmt.Sprintf("%s/%s", pipelineCtx.Workspace, p.Task.DockerFile), ImageName: p.Task.Image, @@ -194,7 +202,7 @@ func (p *DockerBuildPlugin) Run(ctx context.Context, pipelineTask *task.Task, pi return } - job, err := buildJob(p.Type(), pipelineTask.ConfigPayload.Release.PredatorImage, p.JobName, serviceName, setting.MinRequest, pipelineCtx, pipelineTask, []*task.RegistryNamespace{}) + job, err := buildJob(p.Type(), pipelineTask.ConfigPayload.Release.PredatorImage, p.JobName, serviceName, "", pipelineTask.ConfigPayload.Build.KubeNamespace, setting.MinRequest, setting.MinRequestSpec, pipelineCtx, pipelineTask, []*task.RegistryNamespace{}) if err != nil { msg := fmt.Sprintf("create build job context error: %v", err) p.Log.Error(msg) @@ -216,7 +224,7 @@ func (p *DockerBuildPlugin) Run(ctx context.Context, pipelineTask *task.Task, pi // Wait ... func (p *DockerBuildPlugin) Wait(ctx context.Context) { - status := waitJobEnd(ctx, p.TaskTimeout(), p.KubeNamespace, p.JobName, p.kubeClient, p.Log) + status := waitJobEnd(ctx, p.TaskTimeout(), p.KubeNamespace, p.JobName, p.kubeClient, p.clientset, p.restConfig, p.Log) p.SetStatus(status) } @@ -242,7 +250,7 @@ func (p *DockerBuildPlugin) Complete(ctx context.Context, pipelineTask *task.Tas }() // 保存实时日志到s3 - err := saveContainerLog(pipelineTask, p.KubeNamespace, p.FileName, jobLabel, p.kubeClient) + err := saveContainerLog(pipelineTask, p.KubeNamespace, "", p.FileName, jobLabel, p.kubeClient) if err != nil { p.Log.Error(err) p.Task.Error = err.Error() diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/extension.go b/pkg/microservice/warpdrive/core/service/taskplugin/extension.go new file mode 100644 index 0000000000000000000000000000000000000000..9ebeb6656b565189ae6579fe45d753f35875a29a --- /dev/null +++ b/pkg/microservice/warpdrive/core/service/taskplugin/extension.go @@ -0,0 +1,270 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package taskplugin + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "go.uber.org/zap" + "sigs.k8s.io/controller-runtime/pkg/client" + + configbase "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/warpdrive/config" + "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types/task" + "github.com/koderover/zadig/pkg/tool/httpclient" + krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client" +) + +const ( + ExtensionTaskTimeout = 60 * 60 * 1 // 60 minutes + ZadigEvent = "X-Zadig-Event" + EventName = "Workflow" +) + +// InitializeExtensionTaskPlugin to initialize build task plugin, and return reference +func InitializeExtensionTaskPlugin(taskType config.TaskType) TaskPlugin { + return &ExtensionTaskPlugin{ + Name: taskType, + kubeClient: krkubeclient.Client(), + } +} + +// ExtensionTaskPlugin is Plugin, name should be compatible with task type +type ExtensionTaskPlugin struct { + Name config.TaskType + KubeNamespace string + JobName string + FileName string + kubeClient client.Client + Task *task.Extension + Log *zap.SugaredLogger + cancel context.CancelFunc + ack func() + pipelineName string + taskId int64 +} + +func (p *ExtensionTaskPlugin) SetAckFunc(ack func()) { + p.ack = ack +} + +// Init ... +func (p *ExtensionTaskPlugin) Init(jobname, filename string, xl *zap.SugaredLogger) { + p.JobName = jobname + p.Log = xl + p.FileName = filename +} + +func (p *ExtensionTaskPlugin) Type() config.TaskType { + return p.Name +} + +// Status ... +func (p *ExtensionTaskPlugin) Status() config.Status { + return p.Task.TaskStatus +} + +// SetStatus ... +func (p *ExtensionTaskPlugin) SetStatus(status config.Status) { + p.Task.TaskStatus = status +} + +// TaskTimeout ... +func (p *ExtensionTaskPlugin) TaskTimeout() int { + if p.Task.Timeout == 0 { + p.Task.Timeout = ExtensionTaskTimeout + } else { + if !p.Task.IsRestart { + p.Task.Timeout = p.Task.Timeout * 60 + } + } + return p.Task.Timeout +} + +func (p *ExtensionTaskPlugin) SetExtensionStatusCompleted(status config.Status) { + p.Task.TaskStatus = status + p.Task.EndTime = time.Now().Unix() +} + +func (p *ExtensionTaskPlugin) Run(ctx context.Context, pipelineTask *task.Task, pipelineCtx *task.PipelineCtx, serviceName string) { + var ( + err error + body []byte + ) + defer func() { + if err != nil { + p.Log.Error(err) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = err.Error() + return + } + }() + p.pipelineName = pipelineTask.PipelineName + p.taskId = pipelineTask.TaskID + p.Log.Infof("succeed to create extension task %s", p.JobName) + _, p.cancel = context.WithCancel(context.Background()) + httpClient := httpclient.New( + httpclient.SetHostURL(p.Task.URL), + ) + url := p.Task.Path + webhookPayload := &task.WebhookPayload{ + EventName: "workflow", + ProjectName: pipelineTask.ProductName, + TaskName: pipelineTask.PipelineName, + TaskID: pipelineTask.TaskID, + ServiceInfos: p.Task.ServiceInfos, + Creator: pipelineTask.TaskCreator, + WorkflowStatus: getPipelineStatus(pipelineTask), + } + body, err = json.Marshal(webhookPayload) + if err != nil { + return + } + + p.Task.Payload = string(body) + headers := make(map[string]string) + for _, header := range p.Task.Headers { + headers[header.Key] = header.Value + } + headers[ZadigEvent] = EventName + response, err := httpClient.Post(url, httpclient.SetHeaders(headers), httpclient.SetBody(body)) + if err != nil { + return + } + responseBody := string(response.Body()) + responseCode := response.StatusCode() + p.Task.ResponseBody = &responseBody + p.Task.ResponseCode = &responseCode + if !p.Task.IsCallback { + p.SetExtensionStatusCompleted(config.StatusPassed) + } +} + +func getPipelineStatus(pipelineTask *task.Task) config.Status { + for _, stage := range pipelineTask.Stages { + if stage.Status == config.StatusFailed || stage.Status == config.StatusCancelled || stage.Status == config.StatusTimeout { + return stage.Status + } + } + return config.StatusPassed +} + +// Wait ... +func (p *ExtensionTaskPlugin) Wait(ctx context.Context) { + timeout := time.After(time.Duration(p.TaskTimeout()) * time.Second) + defer p.cancel() + + for { + select { + case <-ctx.Done(): + p.Task.TaskStatus = config.StatusCancelled + return + case <-timeout: + p.Task.TaskStatus = config.StatusTimeout + p.Task.Error = "timeout" + return + default: + time.Sleep(time.Second * 3) + callbackPayloadObj, _ := p.getCallbackObj(p.taskId, p.pipelineName) + if callbackPayloadObj != nil { + if callbackPayloadObj.Status == "success" { + p.Task.TaskStatus = config.StatusPassed + return + } + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = callbackPayloadObj.StatusMessage + return + } + + if p.IsTaskDone() { + return + } + } + } +} + +func (p *ExtensionTaskPlugin) getCallbackObj(taskID int64, pipelineName string) (*task.CallbackPayloadObj, error) { + url := fmt.Sprintf("/api/workflow/workflowtask/callback/id/%d/name/%s", taskID, pipelineName) + httpClient := httpclient.New( + httpclient.SetHostURL(configbase.AslanServiceAddress()), + ) + + CallbackPayloadObj := new(task.CallbackPayloadObj) + _, err := httpClient.Get(url, httpclient.SetResult(&CallbackPayloadObj)) + if err != nil { + return nil, err + } + return CallbackPayloadObj, nil +} + +// Complete ... +func (p *ExtensionTaskPlugin) Complete(ctx context.Context, pipelineTask *task.Task, serviceName string) { +} + +// SetTask ... +func (p *ExtensionTaskPlugin) SetTask(t map[string]interface{}) error { + task, err := ToExtensionTask(t) + if err != nil { + return err + } + p.Task = task + return nil +} + +// GetTask ... +func (p *ExtensionTaskPlugin) GetTask() interface{} { + return p.Task +} + +// IsTaskDone ... +func (p *ExtensionTaskPlugin) IsTaskDone() bool { + if p.Task.TaskStatus != config.StatusCreated && p.Task.TaskStatus != config.StatusRunning { + return true + } + return false +} + +// IsTaskFailed ... +func (p *ExtensionTaskPlugin) IsTaskFailed() bool { + if p.Task.TaskStatus == config.StatusFailed || p.Task.TaskStatus == config.StatusTimeout || p.Task.TaskStatus == config.StatusCancelled { + return true + } + return false +} + +// SetStartTime ... +func (p *ExtensionTaskPlugin) SetStartTime() { + p.Task.StartTime = time.Now().Unix() +} + +// SetEndTime ... +func (p *ExtensionTaskPlugin) SetEndTime() { + p.Task.EndTime = time.Now().Unix() +} + +// IsTaskEnabled ... +func (p *ExtensionTaskPlugin) IsTaskEnabled() bool { + return p.Task.Enabled +} + +// ResetError ... +func (p *ExtensionTaskPlugin) ResetError() { + p.Task.Error = "" +} diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/github/status.go b/pkg/microservice/warpdrive/core/service/taskplugin/github/status.go index 8fdc8b31d9b2bab9c0ff616e4098ce8a0b6b415b..b39985d7745d2180e159bcb479246f5f06598e4d 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/github/status.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/github/status.go @@ -47,7 +47,7 @@ type StatusOptions struct { } func (c *Client) UpdateCheckStatus(opt *StatusOptions) error { - sc := setting.ProductName + sc := setting.ProductName + "/" + opt.PipeName _, err := c.CreateStatus( context.TODO(), opt.Owner, opt.Repo, opt.Ref, &github.RepoStatus{ diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/jenkins_plugin.go b/pkg/microservice/warpdrive/core/service/taskplugin/jenkins_plugin.go index 5e38031fa41e5ab7d962987523b5ac872facf7cc..b88b202d43fe7c735e1068c435542d20392d9ffc 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/jenkins_plugin.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/jenkins_plugin.go @@ -24,6 +24,8 @@ import ( "github.com/bndr/gojenkins" "go.uber.org/zap" yaml "gopkg.in/yaml.v3" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/koderover/zadig/pkg/microservice/jenkinsplugin/core/service" @@ -44,6 +46,8 @@ func InitializeJenkinsBuildPlugin(taskType config.TaskType) TaskPlugin { return &JenkinsBuildPlugin{ Name: taskType, kubeClient: krkubeclient.Client(), + clientset: krkubeclient.Clientset(), + restConfig: krkubeclient.RESTConfig(), } } @@ -52,6 +56,8 @@ type JenkinsBuildPlugin struct { Name config.TaskType KubeNamespace string kubeClient client.Client + clientset kubernetes.Interface + restConfig *rest.Config JobName string FileName string Task *task.JenkinsBuild @@ -157,8 +163,7 @@ func (j *JenkinsBuildPlugin) Run(ctx context.Context, pipelineTask *task.Task, p } j.Log.Infof("succeed to create cm for jenkins build job %s", j.JobName) - j.Log.Infof("JenkinsBuildConfig : %+v", pipelineTask.ConfigPayload.JenkinsBuildConfig) - job, err := buildJob(j.Type(), pipelineTask.ConfigPayload.JenkinsBuildConfig.JenkinsBuildImage, j.JobName, serviceName, setting.MinRequest, pipelineCtx, pipelineTask, []*task.RegistryNamespace{}) + job, err := buildJob(j.Type(), pipelineTask.ConfigPayload.JenkinsBuildConfig.JenkinsBuildImage, j.JobName, serviceName, "", pipelineTask.ConfigPayload.Build.KubeNamespace, setting.MinRequest, setting.MinRequestSpec, pipelineCtx, pipelineTask, j.Task.Registries) if err != nil { msg := fmt.Sprintf("create jenkins build job context error: %v", err) j.Log.Error(msg) @@ -175,6 +180,15 @@ func (j *JenkinsBuildPlugin) Run(ctx context.Context, pipelineTask *task.Task, p return } job.Namespace = j.KubeNamespace + + // Set the access permissions of the private mirror warehouse integrated into KodeRover to the namespace + if err := createOrUpdateRegistrySecrets(j.KubeNamespace, pipelineTask.ConfigPayload.RegistryID, j.Task.Registries, j.kubeClient); err != nil { + msg := fmt.Sprintf("create secret error: %v", err) + j.Log.Error(msg) + j.Task.TaskStatus = config.StatusFailed + j.Task.Error = msg + return + } if err := updater.CreateJob(job, j.kubeClient); err != nil { msg := fmt.Sprintf("create jenkins build job error: %v", err) j.Log.Error(msg) @@ -187,18 +201,30 @@ func (j *JenkinsBuildPlugin) Run(ctx context.Context, pipelineTask *task.Task, p // Wait ... func (j *JenkinsBuildPlugin) Wait(ctx context.Context) { - jobStatus := waitJobEnd(ctx, j.TaskTimeout(), j.KubeNamespace, j.JobName, j.kubeClient, j.Log) + jobStatus := waitJobEnd(ctx, j.TaskTimeout(), j.KubeNamespace, j.JobName, j.kubeClient, j.clientset, j.restConfig, j.Log) jenkinsClient, err := gojenkins.CreateJenkins(nil, j.Task.JenkinsIntegration.URL, j.Task.JenkinsIntegration.Username, j.Task.JenkinsIntegration.Password).Init(ctx) if err != nil { j.SetStatus(config.StatusFailed) + msg := fmt.Sprintf("failed to create jenkins client, error: %s", err) + j.Log.Error(msg) + j.Task.Error = msg + return } job, err := jenkinsClient.GetJob(ctx, j.Task.JenkinsBuildArgs.JobName) if err != nil { j.SetStatus(config.StatusFailed) + msg := fmt.Sprintf("failed to get jenkins job, error: %s", err) + j.Log.Error(msg) + j.Task.Error = msg + return } build, err := job.GetLastCompletedBuild(ctx) if err != nil { j.SetStatus(config.StatusFailed) + msg := fmt.Sprintf("failed to get last completed build, error: %s", err) + j.Log.Error(msg) + j.Task.Error = msg + return } jenkinsBuildStatus := j.matchStatus(jobStatus, build.GetResult()) j.SetStatus(jenkinsBuildStatus) @@ -234,7 +260,7 @@ func (j *JenkinsBuildPlugin) Complete(ctx context.Context, pipelineTask *task.Ta }() // 保存实时日志到s3 - err := saveContainerLog(pipelineTask, j.KubeNamespace, j.FileName, jobLabel, j.kubeClient) + err := saveContainerLog(pipelineTask, j.KubeNamespace, "", j.FileName, jobLabel, j.kubeClient) if err != nil { j.Log.Error(err) j.Task.Error = err.Error() diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/jira.go b/pkg/microservice/warpdrive/core/service/taskplugin/jira.go index f544f50e4e4176d6e792c638ca517e31c0ccc393..7ad0c063d767edbec0292eb6789c9ea0b4c84b9e 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/jira.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/jira.go @@ -26,12 +26,11 @@ import ( "github.com/xanzy/go-gitlab" "go.uber.org/zap" - configbase "github.com/koderover/zadig/pkg/config" "github.com/koderover/zadig/pkg/microservice/warpdrive/config" wdgithub "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/taskplugin/github" "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types/task" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/poetry" + "github.com/koderover/zadig/pkg/shared/client/systemconfig" "github.com/koderover/zadig/pkg/tool/jira" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/util" @@ -166,17 +165,35 @@ func (p *JiraPlugin) Run(ctx context.Context, pipelineTask *task.Task, pipelineC repoCommits = append(repoCommits, list...) opt.Page = resp.NextPage } - for _, rc := range repoCommits { - // parse commit message - if rc != nil && rc.Message != "" { - keys := util.GetJiraKeys(rc.Message) - if len(keys) > 0 { - jiraKeys = append(jiraKeys, keys...) - } + + } else { + opts := &gitlab.ListCommitsOptions{ + RefName: &build.Branch, + ListOptions: gitlab.ListOptions{ + PerPage: 1, + Page: 1, + }, + } + commits, _, err := gitlabCli.Commits.ListCommits(fmt.Sprintf("%s/%s", build.RepoOwner, build.RepoName), opts) + if err != nil { + p.Log.Errorf("gitlab ListCommits [%s/%s] error: %v", build.RepoOwner, build.RepoName, err) + p.Task.TaskStatus = config.StatusSkipped + p.Task.Error = err.Error() + return + } + repoCommits = append(repoCommits, commits...) + } + + for _, rc := range repoCommits { + // parse commit message + if rc != nil && rc.Message != "" { + keys := util.GetJiraKeys(rc.Message) + if len(keys) > 0 { + jiraKeys = append(jiraKeys, keys...) } } } - } else { + } else if build.Source == setting.SourceFromGithub { httpsAddr := "" if pipelineTask.ConfigPayload.Proxy.EnableRepoProxy && pipelineTask.ConfigPayload.Proxy.Type == "http" { httpsAddr = pipelineTask.ConfigPayload.Proxy.GetProxyURL() @@ -231,6 +248,30 @@ func (p *JiraPlugin) Run(ctx context.Context, pipelineTask *task.Task, pipelineC return } + for _, rc := range commits { + // parse commit message + if rc.Commit != nil && rc.Commit.Message != nil { + keys := util.GetJiraKeys(*rc.Commit.Message) + if len(keys) > 0 { + jiraKeys = append(jiraKeys, keys...) + } + } + } + } else { + opt := &github.CommitsListOptions{ + SHA: build.Branch, + ListOptions: github.ListOptions{ + Page: 1, + PerPage: 1, + }, + } + commits, _, err := gh.Client.Repositories.ListCommits(context.TODO(), build.RepoOwner, build.RepoName, opt) + if err != nil { + p.Log.Errorf("github ListCommits [%s/%s] error: %v", build.RepoOwner, build.RepoName, err) + p.Task.TaskStatus = config.StatusSkipped + p.Task.Error = err.Error() + return + } for _, rc := range commits { // parse commit message if rc.Commit != nil && rc.Commit.Message != nil { @@ -340,8 +381,7 @@ func (p *JiraPlugin) SetEndTime() { func (p *JiraPlugin) getJiraIssue(pipelineTask *task.Task, key string) (*task.JiraIssue, error) { jiraIssue := new(task.JiraIssue) - poetryClient := poetry.New(configbase.PoetryServiceAddress(), config.PoetryAPIRootKey()) - jiraInfo, err := poetryClient.GetJiraInfo() + jiraInfo, err := systemconfig.New().GetJiraInfo() if err != nil { return nil, fmt.Errorf("getJiraInfo [%s] error: %v", key, err) } diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/job.go b/pkg/microservice/warpdrive/core/service/taskplugin/job.go index d0ddc2ad15cbb9daec45b8599c192c6687b57525..ed078f42608ff9a5ee7db14e767ece9f6a6b9e1b 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/job.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/job.go @@ -25,7 +25,9 @@ import ( "net" "os" "path" + "regexp" "sort" + "strconv" "strings" "time" @@ -35,6 +37,8 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/koderover/zadig/pkg/microservice/warpdrive/config" @@ -42,14 +46,16 @@ import ( "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types" "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types/task" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" "github.com/koderover/zadig/pkg/shared/kube/wrapper" - krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client" "github.com/koderover/zadig/pkg/tool/kube/containerlog" "github.com/koderover/zadig/pkg/tool/kube/getter" "github.com/koderover/zadig/pkg/tool/kube/podexec" "github.com/koderover/zadig/pkg/tool/kube/updater" + kubeutil "github.com/koderover/zadig/pkg/tool/kube/util" "github.com/koderover/zadig/pkg/tool/log" s3tool "github.com/koderover/zadig/pkg/tool/s3" + commontypes "github.com/koderover/zadig/pkg/types" "github.com/koderover/zadig/pkg/util" ) @@ -57,10 +63,15 @@ const ( defaultSecretEmail = "bot@koderover.com" PredatorPlugin = "predator-plugin" JenkinsPlugin = "jenkins-plugin" -) - -const ( - registrySecretSuffix = "-registry-secret" + PackagerPlugin = "packager-plugin" + NormalSchedule = "normal" + RequiredSchedule = "required" + PreferredSchedule = "preferred" + + registrySecretSuffix = "-registry-secret" + ResourceServer = "resource-server" + DindServer = "dind" + KoderoverAgentNamespace = "koderover-agent" ) func saveFile(src io.Reader, localFile string) error { @@ -75,7 +86,7 @@ func saveFile(src io.Reader, localFile string) error { return err } -func saveContainerLog(pipelineTask *task.Task, namespace, fileName string, jobLabel *JobLabel, kubeClient client.Client) error { +func saveContainerLog(pipelineTask *task.Task, namespace, clusterID, fileName string, jobLabel *JobLabel, kubeClient client.Client) error { selector := labels.Set(getJobLabels(jobLabel)).AsSelector() pods, err := getter.ListPods(namespace, selector, kubeClient) if err != nil { @@ -95,10 +106,17 @@ func saveContainerLog(pipelineTask *task.Task, namespace, fileName string, jobLa sort.SliceStable(pods, func(i, j int) bool { return pods[i].CreationTimestamp.Before(&pods[j].CreationTimestamp) }) - if err := containerlog.GetContainerLogs(namespace, pods[0].Name, pods[0].Spec.Containers[0].Name, false, int64(0), buf, krkubeclient.Clientset()); err != nil { + + clientSet, err := kubeclient.GetClientset(pipelineTask.ConfigPayload.HubServerAddr, clusterID) + if err != nil { + log.Errorf("saveContainerLog, get client set error: %s", err) return err } + if err := containerlog.GetContainerLogs(namespace, pods[0].Name, pods[0].Spec.Containers[0].Name, false, int64(0), buf, clientSet); err != nil { + return fmt.Errorf("failed to get container logs: %s", err) + } + if tempFileName, err := util.GenerateTmpFile(); err == nil { defer func() { _ = os.Remove(tempFileName) @@ -106,12 +124,13 @@ func saveContainerLog(pipelineTask *task.Task, namespace, fileName string, jobLa if err = saveFile(buf, tempFileName); err == nil { var store *s3.S3 if store, err = s3.NewS3StorageFromEncryptedURI(pipelineTask.StorageURI); err != nil { + log.Errorf("failed to Create S3 endpoint from Encrypted URI: %s, the error is: %s ", pipelineTask.StorageURI, err) return err } if store.Subfolder != "" { - store.Subfolder = fmt.Sprintf("%s/%s/%d/%s", store.Subfolder, pipelineTask.PipelineName, pipelineTask.TaskID, "log") + store.Subfolder = fmt.Sprintf("%s/%s/%d/%s", store.Subfolder, strings.ToLower(pipelineTask.PipelineName), pipelineTask.TaskID, "log") } else { - store.Subfolder = fmt.Sprintf("%s/%d/%s", pipelineTask.PipelineName, pipelineTask.TaskID, "log") + store.Subfolder = fmt.Sprintf("%s/%d/%s", strings.ToLower(pipelineTask.PipelineName), pipelineTask.TaskID, "log") } forcedPathStyle := true if store.Provider == setting.ProviderSourceAli { @@ -151,7 +170,6 @@ func saveContainerLog(pipelineTask *task.Task, namespace, fileName string, jobLa return nil } -// JobCtxBuilder ... type JobCtxBuilder struct { JobName string ArchiveFile string @@ -172,12 +190,12 @@ func replaceWrapLine(script string) string { // BuildReaperContext builds a yaml func (b *JobCtxBuilder) BuildReaperContext(pipelineTask *task.Task, serviceName string) *types.Context { - ctx := &types.Context{ + APIToken: pipelineTask.ConfigPayload.APIToken, Workspace: b.PipelineCtx.Workspace, CleanWorkspace: b.JobCtx.CleanWorkspace, IgnoreCache: pipelineTask.ConfigPayload.IgnoreCache, - ResetCache: pipelineTask.ConfigPayload.ResetCache, + // ResetCache: pipelineTask.ConfigPayload.ResetCache, Proxy: &types.Proxy{ Type: pipelineTask.ConfigPayload.Proxy.Type, Address: pipelineTask.ConfigPayload.Proxy.Address, @@ -208,6 +226,28 @@ func (b *JobCtxBuilder) BuildReaperContext(pipelineTask *task.Task, serviceName TaskID: pipelineTask.TaskID, ServiceName: serviceName, StorageEndpoint: pipelineTask.StorageEndpoint, + AesKey: pipelineTask.ConfigPayload.AesKey, + UploadEnabled: b.JobCtx.UploadEnabled, + UploadInfo: b.JobCtx.UploadInfo, + } + + // Currently only the build stage will have this info. For the rest we will skip this context + if b.JobCtx.UploadStorageInfo != nil { + ctx.UploadStorageInfo = &commontypes.ObjectStorageInfo{ + Endpoint: b.JobCtx.UploadStorageInfo.Endpoint, + AK: b.JobCtx.UploadStorageInfo.AK, + SK: b.JobCtx.UploadStorageInfo.SK, + Bucket: b.JobCtx.UploadStorageInfo.Bucket, + Insecure: b.JobCtx.UploadStorageInfo.Insecure, + Provider: b.JobCtx.UploadStorageInfo.Provider, + } + } + + if b.PipelineCtx.CacheEnable && !pipelineTask.ConfigPayload.ResetCache { + ctx.CacheEnable = true + ctx.Cache = b.PipelineCtx.Cache + ctx.CacheDirType = b.PipelineCtx.CacheDirType + ctx.CacheUserDir = b.PipelineCtx.CacheUserDir } for _, install := range b.Installs { @@ -228,24 +268,30 @@ func (b *JobCtxBuilder) BuildReaperContext(pipelineTask *task.Task, serviceName for _, build := range b.JobCtx.Builds { repo := &types.Repo{ - Source: build.Source, - Owner: build.RepoOwner, - Name: build.RepoName, - RemoteName: build.RemoteName, - Branch: build.Branch, - PR: build.PR, - Tag: build.Tag, - CheckoutPath: build.CheckoutPath, - SubModules: build.SubModules, - OauthToken: build.OauthToken, - Address: build.Address, - CheckoutRef: build.CheckoutRef, - User: build.Username, - Password: build.Password, + Source: build.Source, + Owner: build.RepoOwner, + Name: build.RepoName, + Namespace: build.RepoNamespace, + RemoteName: build.RemoteName, + Branch: build.Branch, + PR: build.PR, + Tag: build.Tag, + CheckoutPath: build.CheckoutPath, + SubModules: build.SubModules, + OauthToken: build.OauthToken, + Address: build.Address, + CheckoutRef: build.CheckoutRef, + User: build.Username, + Password: build.Password, + EnableProxy: build.EnableProxy, + AuthType: build.AuthType, + SSHKey: build.SSHKey, + PrivateAccessToken: build.PrivateAccessToken, } ctx.Repos = append(ctx.Repos, repo) } + envmaps := make(map[string]string) for _, ev := range b.JobCtx.EnvVars { val := fmt.Sprintf("%s=%s", ev.Key, ev.Value) if ev.IsCredential { @@ -253,6 +299,18 @@ func (b *JobCtxBuilder) BuildReaperContext(pipelineTask *task.Task, serviceName } else { ctx.Envs = append(ctx.Envs, val) } + envmaps[ev.Key] = ev.Value + } + + if b.JobCtx.UploadEnabled { + for _, upload := range b.JobCtx.UploadInfo { + // since the frontend won't change the file path, the backend will have to add it + // use AbsFilePath to avoid change the original user input since a job may be retried + upload.AbsFilePath = fmt.Sprintf("$WORKSPACE/%s", upload.FilePath) + // then we replace it with our env variables + upload.AbsFilePath = replaceEnvWithValue(upload.AbsFilePath, envmaps) + upload.DestinationPath = replaceEnvWithValue(upload.DestinationPath, envmaps) + } } //Support multi build steps @@ -283,10 +341,12 @@ func (b *JobCtxBuilder) BuildReaperContext(pipelineTask *task.Task, serviceName if b.JobCtx.DockerBuildCtx != nil { ctx.DockerBuildCtx = &task.DockerBuildCtx{ - WorkDir: b.JobCtx.DockerBuildCtx.WorkDir, - DockerFile: b.JobCtx.DockerBuildCtx.DockerFile, - ImageName: b.JobCtx.DockerBuildCtx.ImageName, - BuildArgs: b.JobCtx.DockerBuildCtx.BuildArgs, + Source: b.JobCtx.DockerBuildCtx.Source, + WorkDir: b.JobCtx.DockerBuildCtx.WorkDir, + DockerFile: b.JobCtx.DockerBuildCtx.DockerFile, + ImageName: b.JobCtx.DockerBuildCtx.ImageName, + BuildArgs: b.JobCtx.DockerBuildCtx.BuildArgs, + DockerTemplateContent: b.JobCtx.DockerBuildCtx.DockerTemplateContent, } } @@ -296,12 +356,10 @@ func (b *JobCtxBuilder) BuildReaperContext(pipelineTask *task.Task, serviceName ctx.Caches = b.JobCtx.Caches - if b.JobCtx.TestResultPath != "" { - ctx.GinkgoTest = &types.GinkgoTest{ - ResultPath: b.JobCtx.TestResultPath, - TestReportPath: b.JobCtx.TestReportPath, - ArtifactPaths: b.JobCtx.ArtifactPaths, - } + ctx.GinkgoTest = &types.GinkgoTest{ + ResultPath: b.JobCtx.TestResultPath, + TestReportPath: b.JobCtx.TestReportPath, + ArtifactPaths: b.JobCtx.ArtifactPaths, } ctx.StorageEndpoint = pipelineTask.ConfigPayload.S3Storage.Endpoint @@ -309,6 +367,21 @@ func (b *JobCtxBuilder) BuildReaperContext(pipelineTask *task.Task, serviceName ctx.StorageSK = pipelineTask.ConfigPayload.S3Storage.Sk ctx.StorageBucket = pipelineTask.ConfigPayload.S3Storage.Bucket ctx.StorageProvider = pipelineTask.ConfigPayload.S3Storage.Provider + if pipelineTask.ArtifactInfo != nil { + ctx.ArtifactInfo = &types.ArtifactInfo{ + URL: pipelineTask.ArtifactInfo.URL, + WorkflowName: pipelineTask.ArtifactInfo.WorkflowName, + TaskID: pipelineTask.ArtifactInfo.TaskID, + FileName: pipelineTask.ArtifactInfo.FileName, + } + } + if b.JobCtx.ArtifactPath != "" { + ctx.ArtifactPath = b.JobCtx.ArtifactPath + } + + if pipelineTask.Type == config.ScanningType { + ctx.ScannerFlag = true + } return ctx } @@ -341,12 +414,19 @@ const ( // getJobLabels get labels k-v map from JobLabel struct func getJobLabels(jobLabel *JobLabel) map[string]string { - return map[string]string{ + retMap := map[string]string{ jobLabelTaskKey: fmt.Sprintf("%s-%d", strings.ToLower(jobLabel.PipelineName), jobLabel.TaskID), - jobLabelServiceKey: strings.ToLower(jobLabel.ServiceName), + jobLabelServiceKey: strings.ToLower(util.ReturnValidLabelValue(jobLabel.ServiceName)), jobLabelSTypeKey: strings.Replace(jobLabel.TaskType, "_", "-", -1), jobLabelPTypeKey: jobLabel.PipelineType, } + // no need to add labels with empty value to a job + for k, v := range retMap { + if len(v) == 0 { + delete(retMap, k) + } + } + return retMap } func createJobConfigMap(namespace, jobName string, jobLabel *JobLabel, jobCtx string, kubeClient client.Client) error { @@ -371,13 +451,16 @@ func createJobConfigMap(namespace, jobName string, jobLabel *JobLabel, jobCtx st //"s-job": pipelinename-taskid-tasktype-servicename, //"s-task": pipelinename-taskid, //"s-type": tasktype, -func buildJob(taskType config.TaskType, jobImage, jobName, serviceName string, resReq setting.Request, ctx *task.PipelineCtx, pipelineTask *task.Task, registries []*task.RegistryNamespace) (*batchv1.Job, error) { +func buildJob(taskType config.TaskType, jobImage, jobName, serviceName, clusterID, currentNamespace string, resReq setting.Request, resReqSpec setting.RequestSpec, ctx *task.PipelineCtx, pipelineTask *task.Task, registries []*task.RegistryNamespace) (*batchv1.Job, error) { return buildJobWithLinkedNs( taskType, jobImage, jobName, serviceName, + clusterID, + currentNamespace, resReq, + resReqSpec, ctx, pipelineTask, registries, @@ -386,15 +469,24 @@ func buildJob(taskType config.TaskType, jobImage, jobName, serviceName string, r ) } -func buildJobWithLinkedNs(taskType config.TaskType, jobImage, jobName, serviceName string, resReq setting.Request, ctx *task.PipelineCtx, pipelineTask *task.Task, registries []*task.RegistryNamespace, execNs, linkedNs string) (*batchv1.Job, error) { - var reaperBootingScript string +func buildJobWithLinkedNs(taskType config.TaskType, jobImage, jobName, serviceName, clusterID, currentNamespace string, resReq setting.Request, resReqSpec setting.RequestSpec, ctx *task.PipelineCtx, pipelineTask *task.Task, registries []*task.RegistryNamespace, execNs, linkedNs string) (*batchv1.Job, error) { + var ( + reaperBootingScript string + reaperBinaryFile = pipelineTask.ConfigPayload.Release.ReaperBinaryFile + ) + // not local cluster + if clusterID != "" && clusterID != setting.LocalClusterID { + reaperBinaryFile = strings.Replace(reaperBinaryFile, ResourceServer, ResourceServer+".koderover-agent", -1) + } else { + reaperBinaryFile = strings.Replace(reaperBinaryFile, ResourceServer, ResourceServer+"."+currentNamespace, -1) + } - if !strings.Contains(jobImage, PredatorPlugin) && !strings.Contains(jobImage, JenkinsPlugin) { - reaperBootingScript = fmt.Sprintf("curl -m 60 --retry-delay 5 --retry 3 -sL %s -o reaper && chmod +x reaper && mv reaper /usr/local/bin && /usr/local/bin/reaper", pipelineTask.ConfigPayload.Release.ReaperBinaryFile) + if !strings.Contains(jobImage, PredatorPlugin) && !strings.Contains(jobImage, JenkinsPlugin) && !strings.Contains(jobImage, PackagerPlugin) { + reaperBootingScript = fmt.Sprintf("curl -m 10 --retry-delay 3 --retry 3 -sSL %s -o reaper && chmod +x reaper && mv reaper /usr/local/bin && /usr/local/bin/reaper", reaperBinaryFile) if pipelineTask.ConfigPayload.Proxy.EnableApplicationProxy && pipelineTask.ConfigPayload.Proxy.Type == "http" { - reaperBootingScript = fmt.Sprintf("curl -m 60 --retry-delay 5 --retry 3 -sL --proxy %s %s -o reaper && chmod +x reaper && mv reaper /usr/local/bin && /usr/local/bin/reaper", + reaperBootingScript = fmt.Sprintf("curl -m 10 --retry-delay 3 --retry 3 -sSL --proxy %s %s -o reaper && chmod +x reaper && mv reaper /usr/local/bin && /usr/local/bin/reaper", pipelineTask.ConfigPayload.Proxy.GetProxyURL(), - pipelineTask.ConfigPayload.Release.ReaperBinaryFile, + reaperBinaryFile, ) } } @@ -414,11 +506,9 @@ func buildJobWithLinkedNs(taskType config.TaskType, jobImage, jobName, serviceNa }, } for _, reg := range registries { - arr := strings.Split(reg.Namespace, "/") - namespaceInRegistry := arr[len(arr)-1] - secretName := namespaceInRegistry + registrySecretSuffix - if reg.RegType != "" { - secretName = namespaceInRegistry + "-" + reg.RegType + registrySecretSuffix + secretName, err := genRegistrySecretName(reg) + if err != nil { + return nil, fmt.Errorf("failed to generate registry secret name: %s", err) } secret := corev1.LocalObjectReference{ @@ -445,10 +535,9 @@ func buildJobWithLinkedNs(taskType config.TaskType, jobImage, jobName, serviceNa ImagePullSecrets: ImagePullSecrets, Containers: []corev1.Container{ { - ImagePullPolicy: corev1.PullIfNotPresent, + ImagePullPolicy: corev1.PullAlways, Name: labels["s-type"], Image: jobImage, - WorkingDir: pipelineTask.ConfigPayload.S3Storage.Path, Env: []corev1.EnvVar{ { Name: "JOB_CONFIG_FILE", @@ -461,7 +550,9 @@ func buildJobWithLinkedNs(taskType config.TaskType, jobImage, jobName, serviceNa }, }, VolumeMounts: getVolumeMounts(ctx), - Resources: getResourceRequirements(resReq), + Resources: getResourceRequirements(resReq, resReqSpec), + + TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError, }, }, Volumes: getVolumes(jobName), @@ -470,59 +561,107 @@ func buildJobWithLinkedNs(taskType config.TaskType, jobImage, jobName, serviceNa }, } - if !strings.Contains(jobImage, PredatorPlugin) && !strings.Contains(jobImage, JenkinsPlugin) { + if ctx.CacheEnable && ctx.Cache.MediumType == commontypes.NFSMedium { + volumeName := "build-cache" + job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: volumeName, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: ctx.Cache.NFSProperties.PVC, + }, + }, + }) + + mountPath := ctx.CacheUserDir + if ctx.CacheDirType == commontypes.WorkspaceCacheDir { + mountPath = "/workspace" + } + + job.Spec.Template.Spec.Containers[0].VolumeMounts = append(job.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ + Name: volumeName, + MountPath: mountPath, + SubPath: ctx.Cache.NFSProperties.Subpath, + }) + } + + if !strings.Contains(jobImage, PredatorPlugin) && !strings.Contains(jobImage, JenkinsPlugin) && !strings.Contains(jobImage, PackagerPlugin) { job.Spec.Template.Spec.Containers[0].Command = []string{"/bin/sh", "-c"} job.Spec.Template.Spec.Containers[0].Args = []string{reaperBootingScript} } - if linkedNs != "" && execNs != "" && pipelineTask.ConfigPayload.CustomDNSSupported { - job.Spec.Template.Spec.DNSConfig = &corev1.PodDNSConfig{ - Searches: []string{ - linkedNs + ".svc.cluster.local", - execNs + ".svc.cluster.local", - "svc.cluster.local", - "cluster.local", - }, - } - - if addresses, lookupErr := lookupKubeDNSServerHost(); lookupErr == nil { - job.Spec.Template.Spec.DNSPolicy = corev1.DNSNone - // https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-config - // There can be at most 3 IP addresses specified - job.Spec.Template.Spec.DNSConfig.Nameservers = addresses[:Min(3, len(addresses))] - value := "5" - job.Spec.Template.Spec.DNSConfig.Options = []corev1.PodDNSConfigOption{ - {Name: "ndots", Value: &value}, - } - } else { - log.SugaredLogger().Errorf("failed to find ip of kube dns %v", lookupErr) - } + if affinity := addNodeAffinity(clusterID, pipelineTask.ConfigPayload.K8SClusters); affinity != nil { + job.Spec.Template.Spec.Affinity = affinity } + // Note: + // The following logic is valid for the testing task and configures dNSconfig as follows: + // ``` + // dnsConfig: + // nameservers: + // - 192.168.0.157 + // options: + // - name: ndots + // value: "5" + // searches: + // - piggymetrics-env-dev.svc.cluster.local + // - koderover-agent.svc.cluster.local + // - svc.cluster.local + // - cluster.local + // ``` + // + // These are intra-cluster domain names that can be satisfied by default `ClusterFirst` DNSPolicy. + // No one knows why there is such logic, comment it out, run it for a while and then delete it. + // + // if linkedNs != "" && execNs != "" && pipelineTask.ConfigPayload.CustomDNSSupported { + // job.Spec.Template.Spec.DNSConfig = &corev1.PodDNSConfig{ + // Searches: []string{ + // linkedNs + ".svc.cluster.local", + // execNs + ".svc.cluster.local", + // "svc.cluster.local", + // "cluster.local", + // }, + + // if addresses, lookupErr := lookupKubeDNSServerHost(); lookupErr == nil { + // job.Spec.Template.Spec.DNSPolicy = corev1.DNSNone + // // https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-config + // // There can be at most 3 IP addresses specified + // job.Spec.Template.Spec.DNSConfig.Nameservers = addresses[:Min(3, len(addresses))] + // value := "5" + // job.Spec.Template.Spec.DNSConfig.Options = []corev1.PodDNSConfigOption{ + // {Name: "ndots", Value: &value}, + // } + // } else { + // log.SugaredLogger().Errorf("failed to find ip of kube dns %v", lookupErr) + // } + // } + return job, nil } -func createOrUpdateRegistrySecrets(namespace string, registries []*task.RegistryNamespace, kubeClient client.Client) error { - defaultRegistry := &task.RegistryNamespace{ - RegAddr: config.DefaultRegistryAddr(), - AccessKey: config.DefaultRegistryAK(), - SecretKey: config.DefaultRegistrySK(), +// Note: The name of a Secret object must be a valid DNS subdomain name: +// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names +func formatRegistryName(namespaceInRegistry string) (string, error) { + reg, err := regexp.Compile("[^a-zA-Z0-9\\.-]+") + if err != nil { + return "", err + } + processedName := reg.ReplaceAllString(namespaceInRegistry, "") + processedName = strings.ToLower(processedName) + if len(processedName) > 237 { + processedName = processedName[:237] } - registries = append(registries, defaultRegistry) + return processedName, nil +} +func createOrUpdateRegistrySecrets(namespace, registryID string, registries []*task.RegistryNamespace, kubeClient client.Client) error { for _, reg := range registries { if reg.AccessKey == "" { continue } - arr := strings.Split(reg.Namespace, "/") - namespaceInRegistry := arr[len(arr)-1] - secretName := namespaceInRegistry + registrySecretSuffix - if reg.RegType != "" { - secretName = namespaceInRegistry + "-" + reg.RegType + registrySecretSuffix - } - if reg.RegAddr == config.DefaultRegistryAddr() { - secretName = setting.DefaultImagePullSecret + secretName, err := genRegistrySecretName(reg) + if err != nil { + return fmt.Errorf("failed to generate registry secret name: %s", err) } data := make(map[string][]byte) @@ -579,11 +718,6 @@ func getVolumeMounts(ctx *task.PipelineCtx) []corev1.VolumeMount { Name: "job-config", MountPath: ctx.ConfigMapMountDir, }) - resp = append(resp, corev1.VolumeMount{ - Name: "aes-key", - ReadOnly: true, - MountPath: "/etc/encryption", - }) return resp } @@ -600,18 +734,6 @@ func getVolumes(jobName string) []corev1.Volume { }, }, }) - resp = append(resp, corev1.Volume{ - Name: "aes-key", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "zadig-aes-key", - Items: []corev1.KeyToPath{{ - Key: "aesKey", - Path: "aes", - }}, - }, - }, - }) return resp } @@ -621,80 +743,133 @@ func getVolumes(jobName string) []corev1.Volume { // ResReqLow 4 CPU 8 G used by testing module // ResReqMin 2 CPU 2 G used by docker build, release image module // Fallback ResReq 1 CPU 1 G -func getResourceRequirements(resReq setting.Request) corev1.ResourceRequirements { +func getResourceRequirements(resReq setting.Request, resReqSpec setting.RequestSpec) corev1.ResourceRequirements { switch resReq { - case setting.HighRequest: - return corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("16"), - corev1.ResourceMemory: resource.MustParse("32Gi"), - }, - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("4"), - corev1.ResourceMemory: resource.MustParse("4Gi"), - }, - } + return generateResourceRequirements(setting.HighRequest, setting.HighRequestSpec) + case setting.MediumRequest: - return corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("8"), - corev1.ResourceMemory: resource.MustParse("16Gi"), - }, - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("2"), - corev1.ResourceMemory: resource.MustParse("2Gi"), - }, - } + return generateResourceRequirements(setting.MediumRequest, setting.MediumRequestSpec) + case setting.LowRequest: - return corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("4"), - corev1.ResourceMemory: resource.MustParse("8Gi"), - }, - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("1"), - corev1.ResourceMemory: resource.MustParse("1Gi"), - }, - } + return generateResourceRequirements(setting.LowRequest, setting.LowRequestSpec) case setting.MinRequest: - return corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("2"), - corev1.ResourceMemory: resource.MustParse("2Gi"), - }, - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("0.5"), - corev1.ResourceMemory: resource.MustParse("512Mi"), - }, - } + return generateResourceRequirements(setting.MinRequest, setting.MinRequestSpec) + + case setting.DefineRequest: + return generateResourceRequirements(resReq, resReqSpec) + default: + return generateResourceRequirements(setting.DefaultRequest, setting.DefaultRequestSpec) + } +} + +//generateResourceRequirements +//cpu Request:Limit=1:4 +//memory default Request:Limit=1:4 ; if memoryLimit>= 8Gi,Request:Limit=1:8 +func generateResourceRequirements(req setting.Request, reqSpec setting.RequestSpec) corev1.ResourceRequirements { + + if req != setting.DefineRequest { return corev1.ResourceRequirements{ Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("4"), - corev1.ResourceMemory: resource.MustParse("8Gi"), + corev1.ResourceCPU: resource.MustParse(strconv.Itoa(reqSpec.CpuLimit) + setting.CpuUintM), + corev1.ResourceMemory: resource.MustParse(strconv.Itoa(reqSpec.MemoryLimit) + setting.MemoryUintMi), }, Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("1"), - corev1.ResourceMemory: resource.MustParse("1Gi"), + corev1.ResourceCPU: resource.MustParse(strconv.Itoa(reqSpec.CpuReq) + setting.CpuUintM), + corev1.ResourceMemory: resource.MustParse(strconv.Itoa(reqSpec.MemoryReq) + setting.MemoryUintMi), }, } } + + cpuReqInt := reqSpec.CpuLimit / 4 + if cpuReqInt < 1 { + cpuReqInt = 1 + } + memoryReqInt := reqSpec.MemoryLimit / 4 + if memoryReqInt >= 2*1024 { + memoryReqInt = memoryReqInt / 2 + } + if memoryReqInt < 1 { + memoryReqInt = 1 + } + + return corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(strconv.Itoa(reqSpec.CpuLimit) + setting.CpuUintM), + corev1.ResourceMemory: resource.MustParse(strconv.Itoa(reqSpec.MemoryLimit) + setting.MemoryUintMi), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(strconv.Itoa(cpuReqInt) + setting.CpuUintM), + corev1.ResourceMemory: resource.MustParse(strconv.Itoa(memoryReqInt) + setting.MemoryUintMi), + }, + } } //waitJobEnd //Returns job status -func waitJobEnd(ctx context.Context, taskTimeout int, namspace, jobName string, kubeClient client.Client, xl *zap.SugaredLogger) (status config.Status) { - return waitJobEndWithFile(ctx, taskTimeout, namspace, jobName, false, kubeClient, xl) +func waitJobEnd(ctx context.Context, taskTimeout int, namspace, jobName string, kubeClient client.Client, clientset kubernetes.Interface, restConfig *rest.Config, xl *zap.SugaredLogger) (status config.Status) { + return waitJobEndWithFile(ctx, taskTimeout, namspace, jobName, false, kubeClient, clientset, restConfig, xl) +} + +func waitJobReady(ctx context.Context, namespace, jobName string, kubeClient client.Client, xl *zap.SugaredLogger) (status config.Status) { + xl.Infof("Wait job to start: %s/%s", namespace, jobName) + podTimeout := time.After(120 * time.Second) + + var started bool + for { + select { + case <-podTimeout: + return config.StatusTimeout + default: + time.Sleep(time.Second) + + job, found, err := getter.GetJob(namespace, jobName, kubeClient) + if err != nil { + xl.Errorf("Failed to get job `%s` in namespace `%s`: %s", jobName, namespace, err) + continue + } + if !found { + xl.Errorf("Job `%s` not found in namespace `%s`, retry in a second", jobName, namespace) + continue + } + + podLabels := labels.Set{ + jobLabelPTypeKey: job.Labels[jobLabelPTypeKey], + jobLabelServiceKey: job.Labels[jobLabelServiceKey], + jobLabelTaskKey: job.Labels[jobLabelTaskKey], + jobLabelSTypeKey: job.Labels[jobLabelSTypeKey], + } + pods, err := getter.ListPods(namespace, podLabels.AsSelector(), kubeClient) + if err != nil { + xl.Errorf("Failed to get pods in namespace `%s` for Job `%s`: %s", namespace, jobName, err) + continue + } + + for _, pod := range pods { + if pod.Status.Phase == corev1.PodRunning { + started = true + break + } + } + } + + if started { + break + } + } + + return config.StatusRunning } -func waitJobEndWithFile(ctx context.Context, taskTimeout int, namespace, jobName string, checkFile bool, kubeClient client.Client, xl *zap.SugaredLogger) (status config.Status) { +func waitJobEndWithFile(ctx context.Context, taskTimeout int, namespace, jobName string, checkFile bool, kubeClient client.Client, clientset kubernetes.Interface, restConfig *rest.Config, xl *zap.SugaredLogger) (status config.Status) { xl.Infof("wait job to start: %s/%s", namespace, jobName) timeout := time.After(time.Duration(taskTimeout) * time.Second) podTimeout := time.After(120 * time.Second) - // 等待job运行 + + xl.Infof("Timeout of preparing Pod: %s. Timeout of running task: %s.", 120*time.Second, time.Duration(taskTimeout)*time.Second) var started bool for { @@ -746,7 +921,8 @@ func waitJobEndWithFile(ctx context.Context, taskTimeout int, namespace, jobName return config.StatusFailed } - var done bool + var done, exists bool + var jobStatus commontypes.JobStatus for _, pod := range pods { ipod := wrapper.Pod(pod) if ipod.Pending() { @@ -757,9 +933,12 @@ func waitJobEndWithFile(ctx context.Context, taskTimeout int, namespace, jobName } if !ipod.Finished() { - exists, err := checkDogFoodExistsInContainer(namespace, ipod.Name, ipod.ContainerNames()[0]) + jobStatus, exists, err = checkDogFoodExistsInContainer(clientset, restConfig, namespace, ipod.Name, ipod.ContainerNames()[0]) if err != nil { - xl.Infof("failed to check dog food file %s %v", pods[0].Name, err) + // Note: + // Currently, this error indicates "the target Pod cannot be accessed" or "the target Pod can be accessed, but the dog food file does not exist". + // In these two scenarios, `Info` is used to print logs because they are not business semantic exceptions. + xl.Infof("Result of checking dog food file %s: %s", pods[0].Name, err) break } if !exists { @@ -770,8 +949,14 @@ func waitJobEndWithFile(ctx context.Context, taskTimeout int, namespace, jobName } if done { - xl.Infof("dog food is found, stop to wait %s", job.Name) - return config.StatusPassed + xl.Infof("Dog food is found, stop to wait %s. Job status: %s.", job.Name, jobStatus) + + switch jobStatus { + case commontypes.JobFail: + return config.StatusFailed + default: + return config.StatusPassed + } } } else if job.Status.Succeeded != 0 { return config.StatusPassed @@ -782,16 +967,161 @@ func waitJobEndWithFile(ctx context.Context, taskTimeout int, namespace, jobName time.Sleep(time.Second * 1) } - } -func checkDogFoodExistsInContainer(namespace string, pod string, container string) (bool, error) { - _, _, success, err := podexec.ExecWithOptions(podexec.ExecOptions{ - Command: []string{"test", "-f", setting.DogFood}, +func checkDogFoodExistsInContainer(clientset kubernetes.Interface, restConfig *rest.Config, namespace, pod, container string) (commontypes.JobStatus, bool, error) { + stdout, _, success, err := podexec.KubeExec(clientset, restConfig, podexec.ExecOptions{ + Command: []string{"/bin/sh", "-c", fmt.Sprintf("test -f %[1]s && cat %[1]s", setting.DogFood)}, Namespace: namespace, PodName: pod, ContainerName: container, }) - return success, err + return commontypes.JobStatus(stdout), success, err +} + +func addNodeAffinity(clusterID string, K8SClusters []*task.K8SCluster) *corev1.Affinity { + clusterConfig := findClusterConfig(clusterID, K8SClusters) + if clusterConfig == nil { + return nil + } + + if len(clusterConfig.NodeLabels) == 0 { + return nil + } + + switch clusterConfig.Strategy { + case RequiredSchedule: + nodeSelectorTerms := make([]corev1.NodeSelectorTerm, 0) + for _, nodeLabel := range clusterConfig.NodeLabels { + var matchExpressions []corev1.NodeSelectorRequirement + matchExpressions = append(matchExpressions, corev1.NodeSelectorRequirement{ + Key: nodeLabel.Key, + Operator: nodeLabel.Operator, + Values: nodeLabel.Value, + }) + nodeSelectorTerms = append(nodeSelectorTerms, corev1.NodeSelectorTerm{ + MatchExpressions: matchExpressions, + }) + } + + affinity := &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: nodeSelectorTerms, + }, + }, + } + return affinity + case PreferredSchedule: + preferredScheduleTerms := make([]corev1.PreferredSchedulingTerm, 0) + for _, nodeLabel := range clusterConfig.NodeLabels { + var matchExpressions []corev1.NodeSelectorRequirement + matchExpressions = append(matchExpressions, corev1.NodeSelectorRequirement{ + Key: nodeLabel.Key, + Operator: nodeLabel.Operator, + Values: nodeLabel.Value, + }) + nodeSelectorTerm := corev1.NodeSelectorTerm{ + MatchExpressions: matchExpressions, + } + preferredScheduleTerms = append(preferredScheduleTerms, corev1.PreferredSchedulingTerm{ + Weight: 10, + Preference: nodeSelectorTerm, + }) + } + affinity := &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: preferredScheduleTerms, + }, + } + return affinity + default: + return nil + } +} + +func findClusterConfig(clusterID string, K8SClusters []*task.K8SCluster) *task.AdvancedConfig { + for _, K8SCluster := range K8SClusters { + if K8SCluster.ID == clusterID { + return K8SCluster.AdvancedConfig + } + } + return nil +} + +func genRegistrySecretName(reg *task.RegistryNamespace) (string, error) { + if reg.IsDefault { + return setting.DefaultImagePullSecret, nil + } + + arr := strings.Split(reg.Namespace, "/") + namespaceInRegistry := arr[len(arr)-1] + + // for AWS ECR, there are no namespace, thus we need to find the NS from the URI + if namespaceInRegistry == "" { + uriDecipher := strings.Split(reg.RegAddr, ".") + namespaceInRegistry = uriDecipher[0] + } + + filteredName, err := formatRegistryName(namespaceInRegistry) + if err != nil { + return "", err + } + + secretName := filteredName + registrySecretSuffix + if reg.RegType != "" { + secretName = filteredName + "-" + reg.RegType + registrySecretSuffix + } + + return secretName, nil +} + +func replaceEnvWithValue(str string, envs map[string]string) string { + ret := str + for key, value := range envs { + strKey := fmt.Sprintf("$%s", key) + ret = strings.ReplaceAll(ret, strKey, value) + } + return ret +} + +func checkJobExists(ctx context.Context, ns string, jobLabels *JobLabel, kclient client.Client) (jobObj *batchv1.Job, exist bool, err error) { + labelsMap := getJobLabels(jobLabels) + labelSelector := labels.SelectorFromSet(labels.Set(labelsMap)) + + jobList := &batchv1.JobList{} + err = kclient.List(ctx, jobList, &client.ListOptions{ + Namespace: ns, + LabelSelector: labelSelector, + }) + if err != nil || len(jobList.Items) == 0 { + return nil, false, kubeutil.IgnoreNotFoundError(err) + } + + if len(jobList.Items) != 1 { + return nil, true, fmt.Errorf("more than 1 job with labels: %v", labelsMap) + } + + return &(jobList.Items[0]), true, nil +} + +func checkConfigMapExists(ctx context.Context, ns string, jobLabels *JobLabel, kclient client.Client) (cmObj *corev1.ConfigMap, exist bool, err error) { + labelsMap := getJobLabels(jobLabels) + labelSelector := labels.SelectorFromSet(labels.Set(labelsMap)) + + cmList := &corev1.ConfigMapList{} + err = kclient.List(ctx, cmList, &client.ListOptions{ + Namespace: ns, + LabelSelector: labelSelector, + }) + if err != nil || len(cmList.Items) == 0 { + return nil, false, kubeutil.IgnoreNotFoundError(err) + } + + if len(cmList.Items) != 1 { + return nil, true, fmt.Errorf("more than 1 ConfigMap with labels: %v", labelsMap) + } + + return &(cmList.Items[0]), true, nil } diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/job_test.go b/pkg/microservice/warpdrive/core/service/taskplugin/job_test.go index c4b3439b9d7bd0d5b6d9b35ca750ea9b34b252d8..351cd91ae4680919bfe4704aa74a5060162adeca 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/job_test.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/job_test.go @@ -18,17 +18,18 @@ package taskplugin import ( "fmt" + "reflect" "strings" "testing" - "github.com/stretchr/testify/assert" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/koderover/zadig/pkg/microservice/warpdrive/config" "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types/task" "github.com/koderover/zadig/pkg/setting" "github.com/koderover/zadig/pkg/tool/log" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) //Test create configmap named jobname @@ -63,7 +64,7 @@ func TestCreateJobConfigMap(t *testing.T) { TaskType: fmt.Sprintf("%s", config.TaskBuild), } - createJobConfigMap(FakeKubeCli, namespace, jobname, jobLabel, jobCtx) + createJobConfigMap(namespace, jobname, jobLabel, jobCtx, FakeKubeCli) jobConfigmap, err := FakeKubeCli.GetConfigMap(namespace, jobname) assert.Nil(err) assert.Equal(jobname, jobConfigmap.Name) @@ -150,7 +151,7 @@ func TestBuildJob(t *testing.T) { ConfigPayload: &task.ConfigPayload{}, } - _, err := buildJob(config.TaskBuild, jobImage, "test-build-job", "test123", setting.LowRequest, pipelineCtx, pipelineTask, nil) + _, err := buildJob(config.TaskBuild, jobImage, "test-build-job", "test123", setting.LowRequest, setting.LowRequestSpec, pipelineCtx, pipelineTask, nil) log.Error(err) const ( @@ -158,7 +159,7 @@ func TestBuildJob(t *testing.T) { serviceName = "Test123" ) job, err := buildJob(config.TaskBuild, jobImage, jobName, serviceName, - setting.LowRequest, pipelineCtx, pipelineTask, make([]*task.RegistryNamespace, 0)) + setting.LowRequest, setting.LowRequestSpec, pipelineCtx, pipelineTask, make([]*task.RegistryNamespace, 0)) assert.Nil(err) assert.Equal(jobName, job.ObjectMeta.Name) @@ -194,7 +195,7 @@ func TestEnsureDeleteJob(t *testing.T) { ConfigPayload: &task.ConfigPayload{}, } - job, err := buildJob(config.TaskBuild, jobImage, jobname, servicename, setting.LowRequest, pipelineCtx, pipelineTask, nil) + job, err := buildJob(config.TaskBuild, jobImage, jobname, servicename, setting.LowRequest, setting.LowRequestSpec, pipelineCtx, pipelineTask, nil) assert.Nil(err) defer func() { @@ -230,3 +231,63 @@ func TestEnsureDeleteJob(t *testing.T) { assert.Nil(getJob) */ } + +func Test_getResourceRequirements(t *testing.T) { + type args struct { + resReq setting.Request + resReqSpec setting.RequestSpec + } + tests := []struct { + name string + args args + want corev1.ResourceRequirements + }{ + { + name: "high", + args: args{ + resReq: setting.HighRequest, + resReqSpec: setting.HighRequestSpec, + }, + want: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("16000m"), + corev1.ResourceMemory: resource.MustParse("32768Mi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("4000m"), + corev1.ResourceMemory: resource.MustParse("4096Mi"), + }, + }, + }, + { + name: "define", + args: args{ + resReq: setting.DefineRequest, + resReqSpec: setting.RequestSpec{ + CpuLimit: 4000, + MemoryLimit: 2048, + }, + }, + want: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("4000m"), + corev1.ResourceMemory: resource.MustParse("2048Mi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1000m"), + corev1.ResourceMemory: resource.MustParse("512Mi"), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := getResourceRequirements(tt.args.resReq, tt.args.resReqSpec) + fmt.Println("Limit:", got.Limits.Cpu().String(), got.Limits.Memory().String()) + fmt.Println("Memory:", got.Requests.Cpu().String(), got.Requests.Memory().String()) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("getResourceRequirements() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/release_image.go b/pkg/microservice/warpdrive/core/service/taskplugin/release_image.go index 08495bea140a42eb17127d024a9986df23eb901e..1953efb4f264c71e98e6f6ffd024ebda85d0dc48 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/release_image.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/release_image.go @@ -19,11 +19,28 @@ package taskplugin import ( "context" "fmt" + "path/filepath" "strconv" "time" + configbase "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/taskplugin/s3" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" + helmtool "github.com/koderover/zadig/pkg/tool/helmclient" + "github.com/koderover/zadig/pkg/tool/httpclient" + "github.com/koderover/zadig/pkg/tool/kube/getter" + s3tool "github.com/koderover/zadig/pkg/tool/s3" + fsutil "github.com/koderover/zadig/pkg/util/fs" + helmclient "github.com/mittwald/go-helm-client" + "github.com/pkg/errors" + "helm.sh/helm/v3/pkg/releaseutil" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/labels" + "go.uber.org/zap" yaml "gopkg.in/yaml.v3" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/koderover/zadig/pkg/microservice/warpdrive/config" @@ -39,18 +56,28 @@ func InitializeReleaseImagePlugin(taskType config.TaskType) TaskPlugin { return &ReleaseImagePlugin{ Name: taskType, kubeClient: krkubeclient.Client(), + clientset: krkubeclient.Clientset(), + restConfig: krkubeclient.RESTConfig(), + httpClient: httpclient.New( + httpclient.SetHostURL(configbase.AslanServiceAddress()), + ), } } // ReleaseImagePlugin Plugin name should be compatible with task type type ReleaseImagePlugin struct { Name config.TaskType + HubServerAddr string KubeNamespace string JobName string FileName string kubeClient client.Client + clientset kubernetes.Interface + restConfig *rest.Config Task *task.ReleaseImage Log *zap.SugaredLogger + httpClient *httpclient.Client + StorageURI string } func (p *ReleaseImagePlugin) SetAckFunc(func()) { @@ -106,6 +133,8 @@ func (p *ReleaseImagePlugin) TaskTimeout() int { // Run ... func (p *ReleaseImagePlugin) Run(ctx context.Context, pipelineTask *task.Task, pipelineCtx *task.PipelineCtx, serviceName string) { p.KubeNamespace = pipelineTask.ConfigPayload.Build.KubeNamespace + p.HubServerAddr = pipelineTask.ConfigPayload.HubServerAddr + p.StorageURI = pipelineTask.StorageURI // 设置本次运行需要配置 //t.Workspace = fmt.Sprintf("%s/%s", pipelineTask.ConfigPayload.NFS.Path, pipelineTask.PipelineName) releases := make([]task.RepoImage, 0) @@ -117,7 +146,17 @@ func (p *ReleaseImagePlugin) Run(ctx context.Context, pipelineTask *task.Task, p } } - if len(releases) == 0 { + distributes := make([]*task.DistributeInfo, 0) + for _, distribute := range p.Task.DistributeInfo { + if cfg, ok := pipelineTask.ConfigPayload.RepoConfigs[distribute.RepoID]; ok { + distribute.RepoAK = cfg.AccessKey + distribute.RepoSK = cfg.SecretKey + distributes = append(distributes, distribute) + } + } + + if len(distributes) == 0 { + fmt.Println("distribute is 0") return } @@ -135,7 +174,8 @@ func (p *ReleaseImagePlugin) Run(ctx context.Context, pipelineTask *task.Task, p Password: pipelineTask.ConfigPayload.Registry.SecretKey, }, - ReleaseImages: releases, + ReleaseImages: releases, + DistributeInfo: distributes, } jobCtxBytes, err := yaml.Marshal(jobCtx) @@ -176,7 +216,7 @@ func (p *ReleaseImagePlugin) Run(ctx context.Context, pipelineTask *task.Task, p } p.Log.Infof("succeed to create cm for image job %s", p.JobName) - job, err := buildJob(p.Type(), pipelineTask.ConfigPayload.Release.PredatorImage, p.JobName, serviceName, setting.MinRequest, pipelineCtx, pipelineTask, []*task.RegistryNamespace{}) + job, err := buildJob(p.Type(), pipelineTask.ConfigPayload.Release.PredatorImage, p.JobName, serviceName, "", pipelineTask.ConfigPayload.Build.KubeNamespace, setting.MinRequest, setting.MinRequestSpec, pipelineCtx, pipelineTask, []*task.RegistryNamespace{}) if err != nil { msg := fmt.Sprintf("create release image job context error: %v", err) p.Log.Error(msg) @@ -194,6 +234,11 @@ func (p *ReleaseImagePlugin) Run(ctx context.Context, pipelineTask *task.Task, p } job.Namespace = p.KubeNamespace + startTime := time.Now().Unix() + for _, distribute := range p.Task.DistributeInfo { + distribute.DistributeStartTime = startTime + } + if err := updater.CreateJob(job, p.kubeClient); err != nil { msg := fmt.Sprintf("create release image job error: %v", err) p.Log.Error(msg) @@ -206,8 +251,513 @@ func (p *ReleaseImagePlugin) Run(ctx context.Context, pipelineTask *task.Task, p // Wait ... func (p *ReleaseImagePlugin) Wait(ctx context.Context) { - status := waitJobEnd(ctx, p.TaskTimeout(), p.KubeNamespace, p.JobName, p.kubeClient, p.Log) - p.SetStatus(status) + var err error + var status config.Status + defer func() { + if err != nil { + p.Log.Error(err) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = err.Error() + return + } else { + p.SetStatus(config.StatusPassed) + } + }() + status = waitJobEnd(ctx, p.TaskTimeout(), p.KubeNamespace, p.JobName, p.kubeClient, p.clientset, p.restConfig, p.Log) + distributeEndtime := time.Now().Unix() + for _, distribute := range p.Task.DistributeInfo { + distribute.DistributeEndTime = distributeEndtime + distribute.DistributeStatus = string(status) + } + // if the distribution stage failed, then deploy part won't run + if status != config.StatusPassed { + err = errors.New("failed to distribute images to the repository") + return + } + // otherwise, run any deploy subtasks +DistributeLoop: + for _, distribute := range p.Task.DistributeInfo { + if !distribute.DeployEnabled { + continue + } + // set the start time on deployment start. + distribute.DeployStartTime = time.Now().Unix() + distribute.DeployStatus = "running" + if distribute.DeployClusterID != "" { + p.restConfig, err = kubeclient.GetRESTConfig(p.HubServerAddr, distribute.DeployClusterID) + if err != nil { + err = errors.WithMessage(err, "can't get k8s rest config") + distribute.DeployStatus = string(config.StatusFailed) + distribute.DeployEndTime = time.Now().Unix() + continue + } + p.kubeClient, err = kubeclient.GetKubeClient(p.HubServerAddr, distribute.DeployClusterID) + if err != nil { + err = errors.WithMessage(err, "can't init k8s client") + distribute.DeployStatus = string(config.StatusFailed) + distribute.DeployEndTime = time.Now().Unix() + continue + } + } + // k8s deploy type service goes here + if distribute.DeployServiceType != setting.HelmDeployType { + replaced := false + // get servcie info + var ( + serviceInfo *types.ServiceTmpl + selector labels.Selector + ) + serviceInfo, err = p.getService(ctx, distribute.DeployServiceName, distribute.DeployServiceType, p.Task.ProductName, 0) + if err != nil { + // Maybe it is a share service, the entity is not under the project + serviceInfo, err = p.getService(ctx, distribute.DeployServiceName, distribute.DeployServiceType, "", 0) + if err != nil { + err = errors.WithMessage(err, "failed to get service info") + distribute.DeployStatus = string(config.StatusFailed) + distribute.DeployEndTime = time.Now().Unix() + continue + } + } + if serviceInfo.WorkloadType == "" { + selector = labels.Set{setting.ProductLabel: p.Task.ProductName, setting.ServiceLabel: distribute.DeployServiceName}.AsSelector() + + var deployments []*appsv1.Deployment + deployments, err = getter.ListDeployments(distribute.DeployNamespace, selector, p.kubeClient) + if err != nil { + err = errors.WithMessage(err, "failed to get deployment") + distribute.DeployStatus = string(config.StatusFailed) + distribute.DeployEndTime = time.Now().Unix() + continue + } + + var statefulSets []*appsv1.StatefulSet + statefulSets, err = getter.ListStatefulSets(distribute.DeployNamespace, selector, p.kubeClient) + if err != nil { + err = errors.WithMessage(err, "failed to get statefulset") + distribute.DeployStatus = string(config.StatusFailed) + distribute.DeployEndTime = time.Now().Unix() + continue + } + + DeploymentLoop: + for _, deploy := range deployments { + for _, container := range deploy.Spec.Template.Spec.Containers { + if container.Name == distribute.DeployContainerName { + err = updater.UpdateDeploymentImage(deploy.Namespace, deploy.Name, distribute.DeployContainerName, distribute.Image, p.kubeClient) + if err != nil { + err = errors.WithMessagef( + err, + "failed to update container image in %s/deployments/%s/%s", + distribute.DeployNamespace, deploy.Name, container.Name) + distribute.DeployEndTime = time.Now().Unix() + distribute.DeployStatus = string(config.StatusFailed) + p.SetStatus(config.StatusFailed) + continue DistributeLoop + } + replaced = true + break DeploymentLoop + } + } + } + StatefulSetLoop: + for _, sts := range statefulSets { + for _, container := range sts.Spec.Template.Spec.Containers { + if container.Name == distribute.DeployContainerName { + err = updater.UpdateStatefulSetImage(sts.Namespace, sts.Name, distribute.DeployContainerName, distribute.Image, p.kubeClient) + if err != nil { + err = errors.WithMessagef( + err, + "failed to update container image in %s/statefulset/%s/%s", + distribute.DeployNamespace, sts.Name, container.Name) + distribute.DeployEndTime = time.Now().Unix() + distribute.DeployStatus = string(config.StatusFailed) + p.SetStatus(config.StatusFailed) + continue DistributeLoop + } + replaced = true + break StatefulSetLoop + } + } + } + } else { + switch serviceInfo.WorkloadType { + case setting.StatefulSet: + var statefulSet *appsv1.StatefulSet + statefulSet, _, err = getter.GetStatefulSet(distribute.DeployNamespace, distribute.DeployServiceName, p.kubeClient) + if err != nil { + err = errors.WithMessage(err, "failed to get statefulset") + distribute.DeployStatus = string(config.StatusFailed) + distribute.DeployEndTime = time.Now().Unix() + continue + } + for _, container := range statefulSet.Spec.Template.Spec.Containers { + if container.Name == distribute.DeployContainerName { + err = updater.UpdateStatefulSetImage(statefulSet.Namespace, statefulSet.Name, distribute.DeployContainerName, distribute.Image, p.kubeClient) + if err != nil { + err = errors.WithMessagef( + err, + "failed to update container image in %s/statefulsets/%s/%s", + distribute.DeployNamespace, statefulSet.Name, container.Name) + distribute.DeployStatus = string(config.StatusFailed) + distribute.DeployEndTime = time.Now().Unix() + continue DistributeLoop + } + replaced = true + break + } + } + case setting.Deployment: + var deployment *appsv1.Deployment + deployment, _, err = getter.GetDeployment(distribute.DeployNamespace, distribute.DeployServiceName, p.kubeClient) + if err != nil { + err = errors.WithMessage(err, "failed to get deployment") + distribute.DeployStatus = string(config.StatusFailed) + distribute.DeployEndTime = time.Now().Unix() + continue + } + for _, container := range deployment.Spec.Template.Spec.Containers { + if container.Name == distribute.DeployContainerName { + err = updater.UpdateDeploymentImage(deployment.Namespace, deployment.Name, distribute.DeployContainerName, distribute.Image, p.kubeClient) + if err != nil { + err = errors.WithMessagef( + err, + "failed to update container image in %s/deployment/%s/%s", + distribute.DeployNamespace, deployment.Name, container.Name) + distribute.DeployStatus = string(config.StatusFailed) + distribute.DeployEndTime = time.Now().Unix() + continue DistributeLoop + } + replaced = true + break + } + } + } + } + if !replaced { + err = errors.Errorf( + "container %s is not found in resources with label %s", distribute.DeployContainerName, selector) + distribute.DeployStatus = string(config.StatusFailed) + distribute.DeployEndTime = time.Now().Unix() + break + } + // if all is done in one deployment, update its status to success and endtime + distribute.DeployStatus = string(config.StatusPassed) + distribute.DeployEndTime = time.Now().Unix() + } else { + // helm deployment type logic goes here + var ( + productInfo *types.Product + renderChart *types.RenderChart + replacedValuesYaml string + mergedValuesYaml string + replacedMergedValuesYaml string + servicePath string + chartPath string + replaceValuesMap map[string]interface{} + renderInfo *types.RenderSet + helmClient helmclient.Client + ) + + p.Log.Infof("start helm deploy, productName %s serviceName %s containerName %s namespace %s", + p.Task.ProductName, + distribute.DeployServiceName, + distribute.DeployContainerName, + distribute.DeployNamespace) + + productInfo, err = p.getProductInfo(ctx, &EnvArgs{ + EnvName: distribute.DeployEnv, + ProductName: p.Task.ProductName, + }) + if err != nil { + err = errors.WithMessagef( + err, + "failed to get product %s/%s", + distribute.DeployNamespace, + distribute.DeployServiceName) + distribute.DeployStatus = string(config.StatusFailed) + distribute.DeployEndTime = time.Now().Unix() + continue DistributeLoop + } + + renderInfo, err = p.getRenderSet(ctx, productInfo.Render.Name, productInfo.Render.Revision) + if err != nil { + err = errors.WithMessagef( + err, + "failed to get getRenderSet %s/%d", + productInfo.Render.Name, productInfo.Render.Revision) + return + } + + serviceRevisionInProduct := int64(0) + var targetContainer *types.Container + for _, service := range productInfo.GetServiceMap() { + if service.ServiceName == distribute.DeployServiceName { + serviceRevisionInProduct = service.Revision + for _, container := range service.Containers { + if container.Name == distribute.DeployContainerName { + targetContainer = container + break + } + } + break + } + } + + if targetContainer == nil { + err = errors.Errorf("failed to find target container %s from service %s", distribute.DeployContainerName, distribute.DeployServiceName) + distribute.DeployStatus = string(config.StatusFailed) + distribute.DeployEndTime = time.Now().Unix() + continue DistributeLoop + } + + if targetContainer.ImagePath == nil { + err = errors.Errorf("failed to get image path of %s from service %s", distribute.DeployContainerName, distribute.DeployServiceName) + distribute.DeployStatus = string(config.StatusFailed) + distribute.DeployEndTime = time.Now().Unix() + continue DistributeLoop + } + + for _, chartInfo := range renderInfo.ChartInfos { + if chartInfo.ServiceName == distribute.DeployServiceName { + renderChart = chartInfo + break + } + } + + if renderChart == nil { + err = errors.Errorf("failed to update container image in %s/%s，chart not found", + distribute.DeployNamespace, + distribute.DeployServiceName, + ) + distribute.DeployStatus = string(config.StatusFailed) + distribute.DeployEndTime = time.Now().Unix() + continue DistributeLoop + } + + // use revision of service currently applied in environment instead of the latest revision + path, errDownload := p.downloadService(p.Task.ProductName, distribute.DeployServiceName, + p.StorageURI, serviceRevisionInProduct) + if errDownload != nil { + p.Log.Warnf("failed to get chart of revision: %d for service: %s, use latest version", + serviceRevisionInProduct, distribute.DeployServiceName) + path, errDownload = p.downloadService(p.Task.ProductName, distribute.DeployServiceName, + p.StorageURI, 0) + if errDownload != nil { + err = errors.WithMessagef( + errDownload, + "failed to download service %s/%s", + distribute.DeployNamespace, + distribute.DeployServiceName, + ) + distribute.DeployStatus = string(config.StatusFailed) + distribute.DeployEndTime = time.Now().Unix() + continue DistributeLoop + } + } + + chartPath, err = fsutil.RelativeToCurrentPath(path) + if err != nil { + err = errors.WithMessagef( + err, + "failed to get relative path %s", + servicePath, + ) + distribute.DeployStatus = string(config.StatusFailed) + distribute.DeployEndTime = time.Now().Unix() + continue DistributeLoop + } + + serviceValuesYaml := renderChart.ValuesYaml + + // prepare image replace info + validMatchData := getValidMatchData(targetContainer.ImagePath) + + replaceValuesMap, err = assignImageData(distribute.Image, validMatchData) + if err != nil { + err = errors.WithMessagef( + err, + "failed to pase image uri %s/%s", + distribute.DeployNamespace, + distribute.DeployServiceName, + ) + distribute.DeployStatus = string(config.StatusFailed) + distribute.DeployEndTime = time.Now().Unix() + continue DistributeLoop + } + + // replace image into service's values.yaml + replacedValuesYaml, err = replaceImage(serviceValuesYaml, replaceValuesMap) + if err != nil { + err = errors.WithMessagef( + err, + "failed to replace image uri %s/%s", + distribute.DeployNamespace, + distribute.DeployServiceName, + ) + distribute.DeployStatus = string(config.StatusFailed) + distribute.DeployEndTime = time.Now().Unix() + continue DistributeLoop + } + if replacedValuesYaml == "" { + err = errors.Errorf("failed to set new image uri into service's values.yaml %s/%s", + distribute.DeployNamespace, + distribute.DeployServiceName, + ) + distribute.DeployStatus = string(config.StatusFailed) + distribute.DeployEndTime = time.Now().Unix() + continue DistributeLoop + } + + // merge override values and kvs into service's yaml + mergedValuesYaml, err = helmtool.MergeOverrideValues(serviceValuesYaml, renderInfo.DefaultValues, renderChart.GetOverrideYaml(), renderChart.OverrideValues) + if err != nil { + err = errors.WithMessagef( + err, + "failed to merge override values %s", + renderChart.OverrideValues, + ) + distribute.DeployStatus = string(config.StatusFailed) + distribute.DeployEndTime = time.Now().Unix() + continue DistributeLoop + } + + // replace image into final merged values.yaml + replacedMergedValuesYaml, err = replaceImage(mergedValuesYaml, replaceValuesMap) + if err != nil { + err = errors.WithMessagef( + err, + "failed to replace image uri into helm values %s/%s", + distribute.DeployNamespace, + distribute.DeployServiceName, + ) + distribute.DeployStatus = string(config.StatusFailed) + distribute.DeployEndTime = time.Now().Unix() + continue DistributeLoop + } + if replacedMergedValuesYaml == "" { + err = errors.Errorf("failed to set image uri into mreged values.yaml in %s/%s", + distribute.DeployNamespace, + distribute.DeployServiceName, + ) + distribute.DeployStatus = string(config.StatusFailed) + distribute.DeployEndTime = time.Now().Unix() + continue DistributeLoop + } + + p.Log.Infof("final replaced merged values: \n%s", replacedMergedValuesYaml) + + helmClient, err = helmtool.NewClientFromNamespace(distribute.DeployClusterID, distribute.DeployNamespace) + if err != nil { + err = errors.WithMessagef( + err, + "failed to create helm client %s/%s", + distribute.DeployNamespace, + distribute.DeployServiceName, + ) + distribute.DeployStatus = string(config.StatusFailed) + distribute.DeployEndTime = time.Now().Unix() + continue DistributeLoop + } + + releaseName := distribute.ReleaseName + + ensureUpgrade := func() error { + hrs, errHistory := helmClient.ListReleaseHistory(releaseName, 10) + if errHistory != nil { + // list history should not block deploy operation, error will be logged instead of returned + p.Log.Errorf("failed to list release history, release: %s, err: %s", releaseName, errHistory) + return nil + } + if len(hrs) == 0 { + return nil + } + releaseutil.Reverse(hrs, releaseutil.SortByRevision) + rel := hrs[0] + + if rel.Info.Status.IsPending() { + return fmt.Errorf("failed to upgrade release: %s with exceptional status: %s", releaseName, rel.Info.Status) + } + return nil + } + + err = ensureUpgrade() + if err != nil { + distribute.DeployStatus = string(config.StatusFailed) + distribute.DeployEndTime = time.Now().Unix() + continue DistributeLoop + } + + timeOut := p.TaskTimeout() + chartSpec := helmclient.ChartSpec{ + ReleaseName: releaseName, + ChartName: chartPath, + Namespace: distribute.DeployNamespace, + ReuseValues: true, + Version: renderChart.ChartVersion, + ValuesYaml: replacedMergedValuesYaml, + SkipCRDs: false, + UpgradeCRDs: true, + Timeout: time.Second * time.Duration(timeOut), + Wait: true, + Replace: true, + MaxHistory: 10, + } + + done := make(chan bool) + go func(chan bool) { + if _, err = helmClient.InstallOrUpgradeChart(ctx, &chartSpec); err != nil { + err = errors.WithMessagef( + err, + "failed to upgrade helm chart %s/%s", + distribute.DeployNamespace, + distribute.DeployServiceName, + ) + done <- false + } else { + done <- true + } + }(done) + + select { + case <-done: + break + case <-time.After(chartSpec.Timeout + time.Minute): + err = fmt.Errorf("failed to upgrade relase: %s, timeout", chartSpec.ReleaseName) + } + if err != nil { + distribute.DeployStatus = string(config.StatusFailed) + distribute.DeployEndTime = time.Now().Unix() + continue DistributeLoop + } + + //替换环境变量中的chartInfos + for _, chartInfo := range renderInfo.ChartInfos { + if chartInfo.ServiceName == distribute.DeployServiceName { + chartInfo.ValuesYaml = replacedValuesYaml + break + } + } + + // TODO too dangerous to override entire renderset! + err = p.updateRenderSet(ctx, &types.RenderSet{ + Name: renderInfo.Name, + Revision: renderInfo.Revision, + DefaultValues: renderInfo.DefaultValues, + ChartInfos: renderInfo.ChartInfos, + }) + if err != nil { + err = errors.WithMessagef( + err, + "failed to update renderset info %s/%s, renderset %s", + distribute.DeployNamespace, + distribute.DeployServiceName, + renderInfo.Name, + ) + } + distribute.DeployStatus = string(config.StatusPassed) + distribute.DeployEndTime = time.Now().Unix() + } + } } // Complete ... @@ -232,7 +782,7 @@ func (p *ReleaseImagePlugin) Complete(ctx context.Context, pipelineTask *task.Ta }() // 保存实时日志到s3 - err := saveContainerLog(pipelineTask, p.KubeNamespace, p.FileName, jobLabel, p.kubeClient) + err := saveContainerLog(pipelineTask, p.KubeNamespace, "", p.FileName, jobLabel, p.kubeClient) if err != nil { p.Log.Error(err) p.Task.Error = err.Error() @@ -275,7 +825,11 @@ func (p *ReleaseImagePlugin) IsTaskFailed() bool { // SetStartTime ... func (p *ReleaseImagePlugin) SetStartTime() { - p.Task.StartTime = time.Now().Unix() + startTime := time.Now().Unix() + p.Task.StartTime = startTime + for _, distribute := range p.Task.DistributeInfo { + distribute.DistributeStartTime = startTime + } } // SetEndTime ... @@ -292,3 +846,98 @@ func (p *ReleaseImagePlugin) IsTaskEnabled() bool { func (p *ReleaseImagePlugin) ResetError() { p.Task.Error = "" } + +func (p *ReleaseImagePlugin) getService(ctx context.Context, name, serviceType, productName string, revision int64) (*types.ServiceTmpl, error) { + url := fmt.Sprintf("/api/service/services/%s/%s", name, serviceType) + + s := &types.ServiceTmpl{} + _, err := p.httpClient.Get(url, httpclient.SetResult(s), httpclient.SetQueryParams(map[string]string{ + "projectName": productName, + "revision": fmt.Sprintf("%d", revision), + })) + if err != nil { + return nil, err + } + return s, nil +} + +func (p *ReleaseImagePlugin) getProductInfo(ctx context.Context, args *EnvArgs) (*types.Product, error) { + url := fmt.Sprintf("/api/environment/environments/%s/productInfo", args.EnvName) + + prod := &types.Product{} + _, err := p.httpClient.Get(url, httpclient.SetResult(prod), httpclient.SetQueryParam("projectName", args.ProductName)) + if err != nil { + return nil, err + } + return prod, nil +} + +func (p *ReleaseImagePlugin) getRenderSet(ctx context.Context, name string, revision int64) (*types.RenderSet, error) { + url := fmt.Sprintf("/api/project/renders/render/%s/revision/%d", name, revision) + + rs := &types.RenderSet{} + _, err := p.httpClient.Get(url, httpclient.SetResult(rs)) + if err != nil { + return nil, err + } + + return rs, nil +} + +func (p *ReleaseImagePlugin) downloadService(productName, serviceName, storageURI string, revision int64) (string, error) { + logger := p.Log + + fileName := serviceName + if revision > 0 { + fileName = fmt.Sprintf("%s-%d", serviceName, revision) + } + tarball := fmt.Sprintf("%s.tar.gz", fileName) + localBase := configbase.LocalServicePath(productName, serviceName) + tarFilePath := filepath.Join(localBase, tarball) + + exists, err := fsutil.FileExists(tarFilePath) + if err != nil { + return "", err + } + if exists { + return tarFilePath, nil + } + + s3Storage, err := s3.NewS3StorageFromEncryptedURI(storageURI) + if err != nil { + return "", err + } + + s3Storage.Subfolder = filepath.Join(s3Storage.Subfolder, configbase.ObjectStorageServicePath(productName, serviceName)) + forcedPathStyle := true + if s3Storage.Provider == setting.ProviderSourceAli { + forcedPathStyle = false + } + s3Client, err := s3tool.NewClient(s3Storage.Endpoint, s3Storage.Ak, s3Storage.Sk, s3Storage.Insecure, forcedPathStyle) + if err != nil { + p.Log.Errorf("failed to create s3 client, err: %s", err) + return "", err + } + if err = s3Client.Download(s3Storage.Bucket, s3Storage.GetObjectPath(tarball), tarFilePath); err != nil { + logger.Errorf("failed to download file from s3, err: %s", err) + return "", err + } + + exists, err = fsutil.FileExists(tarFilePath) + if err != nil { + return "", err + } + if !exists { + return "", fmt.Errorf("file %s on s3 not found", s3Storage.GetObjectPath(tarball)) + } + + return tarFilePath, nil +} + +func (p *ReleaseImagePlugin) updateRenderSet(ctx context.Context, args *types.RenderSet) error { + url := "/api/project/renders" + + _, err := p.httpClient.Put(url, httpclient.SetBody(args)) + + return err +} diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/scanning.go b/pkg/microservice/warpdrive/core/service/taskplugin/scanning.go new file mode 100644 index 0000000000000000000000000000000000000000..f5addf0a2278b1f8185cffa937f0430fe8b10010 --- /dev/null +++ b/pkg/microservice/warpdrive/core/service/taskplugin/scanning.go @@ -0,0 +1,356 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package taskplugin + +import ( + "context" + "fmt" + "github.com/koderover/zadig/pkg/tool/kube/updater" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "math" + "strings" + "time" + + "go.uber.org/zap" + "gopkg.in/yaml.v3" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + zadigconfig "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/warpdrive/config" + "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types/task" + "github.com/koderover/zadig/pkg/setting" + krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client" +) + +func InitializeScanningTaskPlugin(taskType config.TaskType) TaskPlugin { + return &ScanPlugin{ + Name: taskType, + kubeClient: krkubeclient.Client(), + clientset: krkubeclient.Clientset(), + restConfig: krkubeclient.RESTConfig(), + } +} + +type ScanPlugin struct { + Name config.TaskType + KubeNamespace string + JobName string + FileName string + kubeClient client.Client + clientset kubernetes.Interface + restConfig *rest.Config + Task *task.Scanning + Log *zap.SugaredLogger +} + +func (p *ScanPlugin) SetAckFunc(func()) { +} + +const ( + ScanningTaskTimeout = 60 * 60 // 60 minutes + ScanningTypeSonar = "sonarQube" + ScanningTypeOther = "other" +) + +func (p *ScanPlugin) Init(jobname, filename string, xl *zap.SugaredLogger) { + p.JobName = jobname + p.FileName = filename + p.Log = xl +} + +func (p *ScanPlugin) Type() config.TaskType { + return p.Name +} + +func (p *ScanPlugin) Status() config.Status { + return p.Task.Status +} + +func (p *ScanPlugin) SetStatus(status config.Status) { + p.Task.Status = status +} + +func (p *ScanPlugin) TaskTimeout() int { + if p.Task.Timeout == 0 { + p.Task.Timeout = ScanningTaskTimeout + } + return int(p.Task.Timeout) +} + +func (p *ScanPlugin) Run(ctx context.Context, pipelineTask *task.Task, pipelineCtx *task.PipelineCtx, serviceName string) { + switch p.Task.ClusterID { + case setting.LocalClusterID: + p.KubeNamespace = zadigconfig.Namespace() + default: + p.KubeNamespace = setting.AttachedClusterNamespace + + crClient, clientset, restConfig, err := GetK8sClients(pipelineTask.ConfigPayload.HubServerAddr, p.Task.ClusterID) + if err != nil { + p.Log.Error(err) + p.Task.Status = config.StatusFailed + p.Task.Error = err.Error() + return + } + + p.kubeClient = crClient + p.clientset = clientset + p.restConfig = restConfig + } + + // Since only one repository is supported per scanning, we just hard code it + repo := &task.Repository{ + Source: p.Task.Repos[0].Source, + RepoOwner: p.Task.Repos[0].RepoOwner, + RepoName: p.Task.Repos[0].RepoName, + Branch: p.Task.Repos[0].Branch, + PR: p.Task.Repos[0].PR, + Tag: p.Task.Repos[0].Tag, + OauthToken: p.Task.Repos[0].OauthToken, + Address: p.Task.Repos[0].Address, + Username: p.Task.Repos[0].Username, + Password: p.Task.Repos[0].Password, + EnableProxy: p.Task.Repos[0].EnableProxy, + RemoteName: p.Task.Repos[0].RemoteName, + SubModules: p.Task.Repos[0].SubModules, + } + if repo.RemoteName == "" { + repo.RemoteName = "origin" + } + + jobCtx := JobCtxBuilder{ + JobName: p.JobName, + PipelineCtx: pipelineCtx, + JobCtx: task.JobCtx{ + Builds: []*task.Repository{repo}, + }, + } + + reaperContext := jobCtx.BuildReaperContext(pipelineTask, serviceName) + + // if the scanning task is of sonar type, then we add the sonar parameter to the context + if p.Task.SonarInfo != nil { + reaperContext.SonarParameter = p.Task.Parameter + reaperContext.SonarServer = p.Task.SonarInfo.ServerAddress + reaperContext.SonarLogin = p.Task.SonarInfo.Token + reaperContext.ScannerType = ScanningTypeSonar + } else { + reaperContext.ScannerType = ScanningTypeOther + reaperContext.Scripts = append(reaperContext.Scripts, strings.Split(replaceWrapLine(p.Task.Script), "\n")...) + } + + jobCtxBytes, err := yaml.Marshal(reaperContext) + if err != nil { + msg := fmt.Sprintf("cannot reaper.Context data: %v", err) + p.Log.Error(msg) + p.Task.Status = config.StatusFailed + p.Task.Error = msg + return + } + + jobLabel := &JobLabel{ + PipelineName: pipelineTask.PipelineName, + ServiceName: serviceName, + TaskID: pipelineTask.TaskID, + TaskType: string(p.Type()), + PipelineType: string(pipelineTask.Type), + } + + jobObj, jobExist, err := checkJobExists(ctx, p.KubeNamespace, jobLabel, p.kubeClient) + if err != nil { + msg := fmt.Sprintf("failed to check whether Job exist for %s:%d: %s", pipelineTask.PipelineName, pipelineTask.TaskID, err) + p.Log.Error(msg) + p.Task.Status = config.StatusFailed + p.Task.Error = msg + return + } + + if jobExist { + p.Log.Infof("Job %s:%d eixsts.", pipelineTask.PipelineName, pipelineTask.TaskID) + + p.JobName = jobObj.Name + + // If the code is executed at this point, it indicates that the `wd` instance that executed the Job has been restarted and the + // Job timeout period needs to be corrected. + // + // Rule of reseting timeout: `timeout - (now - start_time_of_job) + compensate_duration` + // For now, `compensate_duration` is 2min. + taskTimeout := p.TaskTimeout() + elaspedTime := time.Now().Unix() - jobObj.Status.StartTime.Time.Unix() + timeout := taskTimeout + 120 - int(elaspedTime) + p.Log.Infof("Timeout before normalization: %d seconds", timeout) + if timeout < 0 { + // That shouldn't happen in theory, but a protection is needed. + timeout = 0 + } + + p.Log.Infof("Timeout after normalization: %d seconds", timeout) + p.tmpSetTaskTimeout(timeout) + } + + _, cmExist, err := checkConfigMapExists(ctx, p.KubeNamespace, jobLabel, p.kubeClient) + if err != nil { + msg := fmt.Sprintf("failed to check whether ConfigMap exist for %s:%d: %s", pipelineTask.PipelineName, pipelineTask.TaskID, err) + p.Log.Error(msg) + p.Task.Status = config.StatusFailed + p.Task.Error = msg + return + } + + if !cmExist { + p.Log.Infof("ConfigMap for Job %s:%d does not exist. Create.", pipelineTask.PipelineName, pipelineTask.TaskID) + + err = createJobConfigMap(p.KubeNamespace, p.JobName, jobLabel, string(jobCtxBytes), p.kubeClient) + if err != nil && !apierrors.IsAlreadyExists(err) { + msg := fmt.Sprintf("createJobConfigMap error: %v", err) + p.Log.Error(msg) + p.Task.Status = config.StatusFailed + p.Task.Error = msg + return + } + } + p.Log.Infof("succeed to create cm for build job %s", p.JobName) + + if !jobExist { + p.Log.Infof("Job %s:%d does not exist. Create.", pipelineTask.PipelineName, pipelineTask.TaskID) + + // search namespace should also include desired namespace + job, err := buildJobWithLinkedNs( + p.Type(), p.Task.ImageInfo, p.JobName, serviceName, p.Task.ClusterID, pipelineTask.ConfigPayload.Test.KubeNamespace, p.Task.ResReq, p.Task.ResReqSpec, pipelineCtx, pipelineTask, p.Task.Registries, + p.KubeNamespace, + // this is a useless field, so we will just leave it empty + "", + ) + + job.Namespace = p.KubeNamespace + + if err != nil { + msg := fmt.Sprintf("create scanning job context error: %v", err) + p.Log.Error(msg) + p.Task.Status = config.StatusFailed + p.Task.Error = msg + return + } + + // 将集成到KodeRover的私有镜像仓库的访问权限设置到namespace中 + if err := createOrUpdateRegistrySecrets(p.KubeNamespace, pipelineTask.ConfigPayload.RegistryID, p.Task.Registries, p.kubeClient); err != nil { + msg := fmt.Sprintf("create secret error: %v", err) + p.Log.Errorf(msg) + p.Task.Status = config.StatusFailed + p.Task.Error = msg + return + } + if err := updater.CreateJob(job, p.kubeClient); err != nil { + msg := fmt.Sprintf("create testing job error: %v", err) + p.Log.Error(msg) + p.Task.Status = config.StatusFailed + p.Task.Error = msg + return + } + } + + p.Log.Infof("succeed to create build job %s", p.JobName) + + p.Task.Status = waitJobReady(ctx, p.KubeNamespace, p.JobName, p.kubeClient, p.Log) +} + +func (p *ScanPlugin) Wait(ctx context.Context) { + status := waitJobEndWithFile(ctx, p.TaskTimeout(), p.KubeNamespace, p.JobName, true, p.kubeClient, p.clientset, p.restConfig, p.Log) + p.SetStatus(status) +} + +func (p *ScanPlugin) tmpSetTaskTimeout(durationInSeconds int) { + p.Task.Timeout = int64(math.Ceil(float64(durationInSeconds) / 60.0)) +} + +func (p *ScanPlugin) Complete(ctx context.Context, pipelineTask *task.Task, serviceName string) { + jobLabel := &JobLabel{ + PipelineName: pipelineTask.PipelineName, + ServiceName: serviceName, + TaskID: pipelineTask.TaskID, + TaskType: string(p.Type()), + PipelineType: string(pipelineTask.Type), + } + + // Clean up tasks that user canceled or timed out. + defer func() { + go func() { + if err := ensureDeleteJob(p.KubeNamespace, jobLabel, p.kubeClient); err != nil { + p.Log.Error(err) + } + + if err := ensureDeleteConfigMap(p.KubeNamespace, jobLabel, p.kubeClient); err != nil { + p.Log.Error(err) + } + }() + }() + + err := saveContainerLog(pipelineTask, p.KubeNamespace, p.Task.ClusterID, p.FileName, jobLabel, p.kubeClient) + if err != nil { + p.Log.Error(err) + p.Task.Error = err.Error() + return + } +} + +func (p *ScanPlugin) SetTask(t map[string]interface{}) error { + task, err := ToScanningTask(t) + if err != nil { + return err + } + p.Task = task + + return nil +} + +func (p *ScanPlugin) GetTask() interface{} { + return p.Task +} + +func (p *ScanPlugin) IsTaskDone() bool { + if p.Task.Status != config.StatusCreated && p.Task.Status != config.StatusRunning { + return true + } + return false +} + +func (p *ScanPlugin) IsTaskFailed() bool { + if p.Task.Status == config.StatusFailed || p.Task.Status == config.StatusTimeout || p.Task.Status == config.StatusCancelled { + return true + } + return false +} + +// since scan is a standalone job, a subtask should not have startTime and endtime + +func (p *ScanPlugin) SetStartTime() { + //p.Task.S = time.Now().Unix() +} + +func (p *ScanPlugin) SetEndTime() { + //p.Task.EndTime = time.Now().Unix() +} + +// scan job is a standalone job so it is always enabled +func (p *ScanPlugin) IsTaskEnabled() bool { + return true +} + +func (p *ScanPlugin) ResetError() { + p.Task.Error = "" +} diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/security.go b/pkg/microservice/warpdrive/core/service/taskplugin/security.go index 43ac5a4ffb361a724519600db59d713b6e70add5..b03ab81ac354210f416e4612289b57ca7d847280 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/security.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/security.go @@ -29,7 +29,6 @@ import ( "github.com/koderover/zadig/pkg/microservice/warpdrive/config" "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types" "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types/task" - "github.com/koderover/zadig/pkg/setting" "github.com/koderover/zadig/pkg/tool/httpclient" ) @@ -39,8 +38,6 @@ func InitializeSecurityPlugin(taskType config.TaskType) TaskPlugin { Name: taskType, errorChan: make(chan error, 1), httpClient: httpclient.New( - httpclient.SetAuthScheme(setting.RootAPIKey), - httpclient.SetAuthToken(config.PoetryAPIRootKey()), httpclient.SetHostURL(configbase.AslanServiceAddress()), ), } diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/taskplugin_suite_test.go b/pkg/microservice/warpdrive/core/service/taskplugin/taskplugin_suite_test.go index 6c9669757e523ff74969d6440a473f1a1d0a4a36..5d87240a9bf8626c8b782d13970326269ca77beb 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/taskplugin_suite_test.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/taskplugin_suite_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package taskplugin_test import ( diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/testing.go b/pkg/microservice/warpdrive/core/service/taskplugin/testing.go index d61273c002fd8eccfe8b4bbaee8d55557dfb536a..b782727ea1b3a87ecbbbb501c92f313a852c2d86 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/testing.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/testing.go @@ -27,8 +27,11 @@ import ( "go.uber.org/zap" "gopkg.in/yaml.v3" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" + zadigconfig "github.com/koderover/zadig/pkg/config" "github.com/koderover/zadig/pkg/microservice/warpdrive/config" "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/taskplugin/s3" "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types" @@ -37,14 +40,16 @@ import ( krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client" "github.com/koderover/zadig/pkg/tool/kube/updater" s3tool "github.com/koderover/zadig/pkg/tool/s3" + commontypes "github.com/koderover/zadig/pkg/types" "github.com/koderover/zadig/pkg/util" ) -//InitializeTestTaskPlugin ... func InitializeTestTaskPlugin(taskType config.TaskType) TaskPlugin { return &TestPlugin{ Name: taskType, kubeClient: krkubeclient.Client(), + clientset: krkubeclient.Clientset(), + restConfig: krkubeclient.RESTConfig(), } } @@ -55,6 +60,8 @@ type TestPlugin struct { JobName string FileName string kubeClient client.Client + clientset kubernetes.Interface + restConfig *rest.Config Task *task.Testing Log *zap.SugaredLogger } @@ -63,34 +70,27 @@ func (p *TestPlugin) SetAckFunc(func()) { } const ( - // TestingV2TaskTimeout ... TestingV2TaskTimeout = 60 * 60 // 60 minutes ) -// Init ... func (p *TestPlugin) Init(jobname, filename string, xl *zap.SugaredLogger) { p.JobName = jobname p.FileName = filename - // SetLogger ... p.Log = xl } -// Type ... func (p *TestPlugin) Type() config.TaskType { return p.Name } -// Status ... func (p *TestPlugin) Status() config.Status { return p.Task.TaskStatus } -// SetStatus ... func (p *TestPlugin) SetStatus(status config.Status) { p.Task.TaskStatus = status } -// TaskTimeout ... func (p *TestPlugin) TaskTimeout() int { if p.Task.Timeout == 0 { p.Task.Timeout = TestingV2TaskTimeout @@ -103,27 +103,72 @@ func (p *TestPlugin) TaskTimeout() int { } func (p *TestPlugin) Run(ctx context.Context, pipelineTask *task.Task, pipelineCtx *task.PipelineCtx, serviceName string) { - p.KubeNamespace = pipelineTask.ConfigPayload.Test.KubeNamespace - // 重置错误信息 + if p.Task.CacheEnable && !pipelineTask.ConfigPayload.ResetCache { + pipelineCtx.CacheEnable = true + pipelineCtx.Cache = p.Task.Cache + pipelineCtx.CacheDirType = p.Task.CacheDirType + pipelineCtx.CacheUserDir = p.Task.CacheUserDir + } else { + pipelineCtx.CacheEnable = false + } + + // TODO: Since the namespace field has been used continuously since v1.10.0, the processing logic related to namespace needs to + // be deleted in v1.11.0. + switch p.Task.ClusterID { + case setting.LocalClusterID: + p.KubeNamespace = zadigconfig.Namespace() + default: + p.KubeNamespace = setting.AttachedClusterNamespace + + crClient, clientset, restConfig, err := GetK8sClients(pipelineTask.ConfigPayload.HubServerAddr, p.Task.ClusterID) + if err != nil { + p.Log.Error(err) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = err.Error() + return + } + + p.kubeClient = crClient + p.clientset = clientset + p.restConfig = restConfig + } + + // not local cluster + var ( + replaceDindServer = "." + DindServer + dockerHost = "" + ) + if p.Task.ClusterID != "" && p.Task.ClusterID != setting.LocalClusterID { + if strings.Contains(pipelineTask.DockerHost, pipelineTask.ConfigPayload.Build.KubeNamespace) { + // replace namespace only + dockerHost = strings.Replace(pipelineTask.DockerHost, pipelineTask.ConfigPayload.Build.KubeNamespace, KoderoverAgentNamespace, 1) + } else { + // add namespace + dockerHost = strings.Replace(pipelineTask.DockerHost, replaceDindServer, replaceDindServer+"."+KoderoverAgentNamespace, 1) + } + } else if p.Task.ClusterID == "" || p.Task.ClusterID == setting.LocalClusterID { + if !strings.Contains(pipelineTask.DockerHost, pipelineTask.ConfigPayload.Build.KubeNamespace) { + // add namespace + dockerHost = strings.Replace(pipelineTask.DockerHost, replaceDindServer, replaceDindServer+"."+pipelineTask.ConfigPayload.Build.KubeNamespace, 1) + } + } + pipelineCtx.DockerHost = dockerHost + // Reset error message. p.Task.Error = "" - // 获取测试相关的namespace var linkedNamespace string var envName string + + workflowName := pipelineTask.PipelineName if pipelineTask.Type == config.SingleType { linkedNamespace = pipelineTask.TaskArgs.Test.Namespace } else if pipelineTask.Type == config.WorkflowType { product := &types.Product{EnvName: pipelineTask.WorkflowArgs.Namespace, ProductName: pipelineTask.WorkflowArgs.ProductTmplName} linkedNamespace = product.ProductName + "-env-" + product.EnvName envName = pipelineTask.WorkflowArgs.Namespace - } - //if linkedNamespace == "" { - // msg := "namespace for testing is empty" - // p.Log.Error(msg) - // p.Task.TaskStatus = task.StatusFailed - // p.Task.Error = msg - // return - //} + workflowName = pipelineTask.WorkflowArgs.WorkflowName + } + p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, &task.KeyVal{Key: "WORKFLOW", Value: workflowName}) namespaceEnvVar := &task.KeyVal{Key: "DEPLOY_ENV", Value: p.KubeNamespace, IsCredential: false} linkedNamespaceEnvVar := &task.KeyVal{Key: "LINKED_ENV", Value: linkedNamespace, IsCredential: false} @@ -138,6 +183,8 @@ func (p *TestPlugin) Run(ctx context.Context, pipelineTask *task.Task, pipelineC p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, namespaceEnvVar) p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, linkedNamespaceEnvVar) p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, envNameEnvVar) + p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, &task.KeyVal{Key: "SERVICE_MODULE", Value: pipelineTask.ServiceName}) + p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, &task.KeyVal{Key: "PROJECT", Value: pipelineTask.ProductName}) var testReportFile string // html 测试报告 fileName := fmt.Sprintf("%s-%s-%d-%s-%s", config.SingleType, pipelineTask.PipelineName, pipelineTask.TaskID, config.TaskTestingV2, pipelineTask.ServiceName) @@ -147,11 +194,19 @@ func (p *TestPlugin) Run(ctx context.Context, pipelineTask *task.Task, pipelineC testReportFile = fmt.Sprintf("%s-%s-%d-%s-%s-html", config.WorkflowType, pipelineTask.PipelineName, pipelineTask.TaskID, config.TaskTestingV2, p.Task.TestModuleName) } else if pipelineTask.Type == config.TestType { fileName = fmt.Sprintf("%s-%s-%d-%s-%s", config.TestType, pipelineTask.PipelineName, pipelineTask.TaskID, config.TaskTestingV2, serviceName) + testReportFile = fmt.Sprintf("%s-%s-%d-%s-%s-html", config.TestType, pipelineTask.PipelineName, pipelineTask.TaskID, config.TaskTestingV2, p.Task.TestModuleName) } fileName = strings.Replace(strings.ToLower(fileName), "_", "-", -1) testReportFile = strings.Replace(strings.ToLower(testReportFile), "_", "-", -1) + // Since we allow users to use custom environment variables, variable resolution is required. + if pipelineCtx.CacheEnable && pipelineCtx.Cache.MediumType == commontypes.NFSMedium && + pipelineCtx.CacheDirType == commontypes.UserDefinedCacheDir { + pipelineCtx.CacheUserDir = p.renderEnv(pipelineCtx.CacheUserDir) + pipelineCtx.Cache.NFSProperties.Subpath = p.renderEnv(pipelineCtx.Cache.NFSProperties.Subpath) + } + jobCtx := JobCtxBuilder{ JobName: p.JobName, PipelineCtx: pipelineCtx, @@ -193,14 +248,11 @@ func (p *TestPlugin) Run(ctx context.Context, pipelineTask *task.Task, pipelineC return } - jobImage := fmt.Sprintf("%s-%s", pipelineTask.ConfigPayload.Release.ReaperImage, p.Task.BuildOS) - if p.Task.ImageFrom == config.ImageFromCustom { - jobImage = p.Task.BuildOS - } + jobImage := getReaperImage(pipelineTask.ConfigPayload.Release.ReaperImage, p.Task.BuildOS, p.Task.ImageFrom) // search namespace should also include desired namespace job, err := buildJobWithLinkedNs( - p.Type(), jobImage, p.JobName, serviceName, p.Task.ResReq, pipelineCtx, pipelineTask, p.Task.Registries, + p.Type(), jobImage, p.JobName, serviceName, p.Task.ClusterID, pipelineTask.ConfigPayload.Test.KubeNamespace, p.Task.ResReq, p.Task.ResReqSpec, pipelineCtx, pipelineTask, p.Task.Registries, p.KubeNamespace, linkedNamespace, ) @@ -224,7 +276,7 @@ func (p *TestPlugin) Run(ctx context.Context, pipelineTask *task.Task, pipelineC } // 将集成到KodeRover的私有镜像仓库的访问权限设置到namespace中 - if err := createOrUpdateRegistrySecrets(p.KubeNamespace, p.Task.Registries, p.kubeClient); err != nil { + if err := createOrUpdateRegistrySecrets(p.KubeNamespace, pipelineTask.ConfigPayload.RegistryID, p.Task.Registries, p.kubeClient); err != nil { p.Log.Errorf("create secret error: %v", err) } if err := updater.CreateJob(job, p.kubeClient); err != nil { @@ -234,15 +286,15 @@ func (p *TestPlugin) Run(ctx context.Context, pipelineTask *task.Task, pipelineC p.Task.Error = msg return } + + p.Task.TaskStatus = waitJobReady(ctx, p.KubeNamespace, p.JobName, p.kubeClient, p.Log) } -// Wait ... func (p *TestPlugin) Wait(ctx context.Context) { - status := waitJobEndWithFile(ctx, p.TaskTimeout(), p.KubeNamespace, p.JobName, true, p.kubeClient, p.Log) + status := waitJobEndWithFile(ctx, p.TaskTimeout(), p.KubeNamespace, p.JobName, true, p.kubeClient, p.clientset, p.restConfig, p.Log) p.SetStatus(status) } -// Complete ... func (p *TestPlugin) Complete(ctx context.Context, pipelineTask *task.Task, serviceName string) { pipelineName := pipelineTask.PipelineName pipelineTaskID := pipelineTask.TaskID @@ -255,18 +307,20 @@ func (p *TestPlugin) Complete(ctx context.Context, pipelineTask *task.Task, serv PipelineType: string(pipelineTask.Type), } - // 日志保存失败与否都清理job + // Clean up tasks that user canceled or timed out. defer func() { - if p.Task.TaskStatus == config.StatusCancelled || p.Task.TaskStatus == config.StatusTimeout { + go func() { if err := ensureDeleteJob(p.KubeNamespace, jobLabel, p.kubeClient); err != nil { p.Log.Error(err) - p.Task.Error = err.Error() } - return - } + + if err := ensureDeleteConfigMap(p.KubeNamespace, jobLabel, p.kubeClient); err != nil { + p.Log.Error(err) + } + }() }() - err := saveContainerLog(pipelineTask, p.KubeNamespace, p.FileName, jobLabel, p.kubeClient) + err := saveContainerLog(pipelineTask, p.KubeNamespace, p.Task.ClusterID, p.FileName, jobLabel, p.kubeClient) if err != nil { p.Log.Error(err) p.Task.Error = err.Error() @@ -287,9 +341,6 @@ func (p *TestPlugin) Complete(ctx context.Context, pipelineTask *task.Task, serv //如果用户配置了测试结果目录需要收集,则下载测试结果,发送到aslan server //Note here: p.Task.TestName目前只有默认值test - if p.Task.JobCtx.TestResultPath == "" { - return - } testReport := new(types.TestReport) if pipelineTask.TestReports == nil { @@ -316,10 +367,12 @@ func (p *TestPlugin) Complete(ctx context.Context, pipelineTask *task.Task, serv } s3client, err := s3tool.NewClient(store.Endpoint, store.Ak, store.Sk, store.Insecure, forcedPathStyle) if err == nil { - prefix := store.GetObjectPath("/") - if files, err := s3client.ListFiles(store.Bucket, prefix, true); err == nil { - if len(files) > 0 { - p.Task.JobCtx.IsHasArtifact = true + if len(p.Task.JobCtx.ArtifactPaths) > 0 { + prefix := store.GetObjectPath("/") + if files, err := s3client.ListFiles(store.Bucket, prefix, true); err == nil { + if len(files) > 0 { + p.Task.JobCtx.IsHasArtifact = true + } } } } @@ -339,7 +392,7 @@ func (p *TestPlugin) Complete(ctx context.Context, pipelineTask *task.Task, serv return } - if p.Task.JobCtx.TestType == setting.FunctionTest { + if p.Task.JobCtx.TestType == setting.FunctionTest && p.Task.JobCtx.TestResultPath != "" { b, err := os.ReadFile(tmpFilename) if err != nil { p.Log.Error(fmt.Sprintf("get test result file error: %v", err)) @@ -462,7 +515,6 @@ func (p *TestPlugin) IsTaskDone() bool { return false } -// IsTaskFailed ... func (p *TestPlugin) IsTaskFailed() bool { if p.Task.TaskStatus == config.StatusFailed || p.Task.TaskStatus == config.StatusTimeout || p.Task.TaskStatus == config.StatusCancelled { return true @@ -470,22 +522,36 @@ func (p *TestPlugin) IsTaskFailed() bool { return false } -// SetStartTime ... func (p *TestPlugin) SetStartTime() { p.Task.StartTime = time.Now().Unix() } -// SetEndTime ... func (p *TestPlugin) SetEndTime() { p.Task.EndTime = time.Now().Unix() } -// IsTaskEnabled .. func (p *TestPlugin) IsTaskEnabled() bool { return p.Task.Enabled } -// ResetError ... func (p *TestPlugin) ResetError() { p.Task.Error = "" } + +// Note: Since there are few environment variables and few variables to be replaced, +// this method is temporarily used. +func (p *TestPlugin) renderEnv(data string) string { + mapper := func(data string) string { + for _, envar := range p.Task.JobCtx.EnvVars { + if data != envar.Key { + continue + } + + return envar.Value + } + + return fmt.Sprintf("$%s", data) + } + + return os.Expand(data, mapper) +} diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/trigger.go b/pkg/microservice/warpdrive/core/service/taskplugin/trigger.go new file mode 100644 index 0000000000000000000000000000000000000000..da50663e909b0974ec479398f8eb765eb5a47213 --- /dev/null +++ b/pkg/microservice/warpdrive/core/service/taskplugin/trigger.go @@ -0,0 +1,298 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package taskplugin + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "go.uber.org/zap" + "sigs.k8s.io/controller-runtime/pkg/client" + + configbase "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/warpdrive/config" + "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/taskplugin/s3" + "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types/task" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/httpclient" + krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client" + "github.com/koderover/zadig/pkg/tool/log" + s3tool "github.com/koderover/zadig/pkg/tool/s3" +) + +const ( + // TriggerTaskTimeout ... + TriggerTaskTimeout = 60 * 60 * 1 // 60 minutes +) + +// InitializeTriggerTaskPlugin to initialize build task plugin, and return reference +func InitializeTriggerTaskPlugin(taskType config.TaskType) TaskPlugin { + return &TriggerTaskPlugin{ + Name: taskType, + kubeClient: krkubeclient.Client(), + } +} + +// TriggerTaskPlugin is Plugin, name should be compatible with task type +type TriggerTaskPlugin struct { + Name config.TaskType + KubeNamespace string + JobName string + FileName string + kubeClient client.Client + Task *task.Trigger + Log *zap.SugaredLogger + cancel context.CancelFunc + ack func() + pipelineName string + taskId int64 +} + +func (p *TriggerTaskPlugin) SetAckFunc(ack func()) { + p.ack = ack +} + +// Init ... +func (p *TriggerTaskPlugin) Init(jobname, filename string, xl *zap.SugaredLogger) { + p.JobName = jobname + p.Log = xl + p.FileName = filename +} + +func (p *TriggerTaskPlugin) Type() config.TaskType { + return p.Name +} + +// Status ... +func (p *TriggerTaskPlugin) Status() config.Status { + return p.Task.TaskStatus +} + +// SetStatus ... +func (p *TriggerTaskPlugin) SetStatus(status config.Status) { + p.Task.TaskStatus = status +} + +// TaskTimeout ... +func (p *TriggerTaskPlugin) TaskTimeout() int { + if p.Task.Timeout == 0 { + p.Task.Timeout = TriggerTaskTimeout + } else { + if !p.Task.IsRestart { + p.Task.Timeout = p.Task.Timeout * 60 + } + } + return p.Task.Timeout +} + +func (p *TriggerTaskPlugin) SetTriggerStatusCompleted(status config.Status) { + p.Task.TaskStatus = status + p.Task.EndTime = time.Now().Unix() +} + +func (p *TriggerTaskPlugin) Run(ctx context.Context, pipelineTask *task.Task, pipelineCtx *task.PipelineCtx, serviceName string) { + var ( + err error + body []byte + artifactPath string + ) + defer func() { + if err != nil { + p.Log.Error(err) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = err.Error() + return + } + }() + p.pipelineName = pipelineTask.PipelineName + p.taskId = pipelineTask.TaskID + p.Log.Infof("succeed to create trigger task %s", p.JobName) + _, p.cancel = context.WithCancel(context.Background()) + httpClient := httpclient.New( + httpclient.SetHostURL(p.Task.URL), + ) + url := p.Task.Path + artifactPath, err = p.getS3Storage(pipelineTask) + if err != nil { + return + } + taskOutput := &task.TaskOutput{ + Type: "object_storage", + Value: artifactPath, + } + webhookPayload := &task.WebhookPayload{ + EventName: "workflow", + ProjectName: pipelineTask.ProductName, + TaskName: pipelineTask.PipelineName, + TaskID: pipelineTask.TaskID, + TaskOutput: []*task.TaskOutput{taskOutput}, + TaskEnvs: pipelineTask.TaskArgs.BuildArgs, + } + body, err = json.Marshal(webhookPayload) + + headers := make(map[string]string) + for _, header := range p.Task.Headers { + headers[header.Key] = header.Value + } + headers[ZadigEvent] = EventName + _, err = httpClient.Post(url, httpclient.SetHeaders(headers), httpclient.SetBody(body)) + if err != nil { + return + } + if !p.Task.IsCallback { + p.SetTriggerStatusCompleted(config.StatusPassed) + } +} + +func (p *TriggerTaskPlugin) getS3Storage(pipelineTask *task.Task) (string, error) { + var err error + var store *s3.S3 + if store, err = s3.NewS3StorageFromEncryptedURI(pipelineTask.StorageURI); err != nil { + log.Errorf("Archive failed to create s3 storage %s", pipelineTask.StorageURI) + return "", err + } + if store.Subfolder != "" { + store.Subfolder = fmt.Sprintf("%s/%s/%d/%s", store.Subfolder, pipelineTask.PipelineName, pipelineTask.TaskID, "artifact") + } else { + store.Subfolder = fmt.Sprintf("%s/%d/%s", pipelineTask.PipelineName, pipelineTask.TaskID, "artifact") + } + forcedPathStyle := true + if store.Provider == setting.ProviderSourceAli { + forcedPathStyle = false + } + s3client, err := s3tool.NewClient(store.Endpoint, store.Ak, store.Sk, store.Insecure, forcedPathStyle) + if err != nil { + return "", err + } + prefix := store.GetObjectPath("") + files, err := s3client.ListFiles(store.Bucket, prefix, true) + if err != nil { + return "", err + } + fileName := "artifact.tar.gz" + if len(files) > 0 { + fileName = files[0] + } + return fmt.Sprintf("%s://%s.%s/%s", store.GetSchema(), store.Bucket, store.Endpoint, fileName), nil +} + +// Wait ... +func (p *TriggerTaskPlugin) Wait(ctx context.Context) { + timeout := time.After(time.Duration(p.TaskTimeout()) * time.Second) + defer p.cancel() + + for { + select { + case <-ctx.Done(): + p.Task.TaskStatus = config.StatusCancelled + return + case <-timeout: + p.Task.TaskStatus = config.StatusTimeout + p.Task.Error = "timeout" + return + default: + time.Sleep(time.Second * 3) + callbackPayloadObj, _ := p.getCallbackObj(p.taskId, p.pipelineName) + if callbackPayloadObj != nil { + p.Task.CallbackType = callbackPayloadObj.Type + p.Task.CallbackPayload = callbackPayloadObj.Payload + if callbackPayloadObj.Status == "success" { + p.Task.TaskStatus = config.StatusPassed + return + } + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = callbackPayloadObj.StatusMessage + return + } + + if p.IsTaskDone() { + return + } + } + } +} + +func (p *TriggerTaskPlugin) getCallbackObj(taskID int64, pipelineName string) (*task.CallbackPayloadObj, error) { + url := fmt.Sprintf("/api/workflow/v3/workflowtask/callback/id/%d/name/%s", taskID, pipelineName) + httpClient := httpclient.New( + httpclient.SetHostURL(configbase.AslanServiceAddress()), + ) + + CallbackPayloadObj := new(task.CallbackPayloadObj) + _, err := httpClient.Get(url, httpclient.SetResult(&CallbackPayloadObj)) + if err != nil { + return nil, err + } + return CallbackPayloadObj, nil +} + +// Complete ... +func (p *TriggerTaskPlugin) Complete(ctx context.Context, pipelineTask *task.Task, serviceName string) { +} + +// SetTask ... +func (p *TriggerTaskPlugin) SetTask(t map[string]interface{}) error { + task, err := ToTriggerTask(t) + if err != nil { + return err + } + p.Task = task + return nil +} + +// GetTask ... +func (p *TriggerTaskPlugin) GetTask() interface{} { + return p.Task +} + +// IsTaskDone ... +func (p *TriggerTaskPlugin) IsTaskDone() bool { + if p.Task.TaskStatus != config.StatusCreated && p.Task.TaskStatus != config.StatusRunning { + return true + } + return false +} + +// IsTaskFailed ... +func (p *TriggerTaskPlugin) IsTaskFailed() bool { + if p.Task.TaskStatus == config.StatusFailed || p.Task.TaskStatus == config.StatusTimeout || p.Task.TaskStatus == config.StatusCancelled { + return true + } + return false +} + +// SetStartTime ... +func (p *TriggerTaskPlugin) SetStartTime() { + p.Task.StartTime = time.Now().Unix() +} + +// SetEndTime ... +func (p *TriggerTaskPlugin) SetEndTime() { + p.Task.EndTime = time.Now().Unix() +} + +// IsTaskEnabled ... +func (p *TriggerTaskPlugin) IsTaskEnabled() bool { + return p.Task.Enabled +} + +// ResetError ... +func (p *TriggerTaskPlugin) ResetError() { + p.Task.Error = "" +} diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/utils.go b/pkg/microservice/warpdrive/core/service/taskplugin/utils.go index 6d41110151f5ad42fcade3424f17542a7daac8a5..4b5c3a917f0a08acbbf970eddaad6ff2be7148cb 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/utils.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/utils.go @@ -21,15 +21,21 @@ import ( "encoding/json" "fmt" "math/rand" + "strconv" "strings" "time" t "github.com/docker/docker/api/types" "github.com/docker/docker/client" "go.uber.org/zap" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + crClient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/koderover/zadig/pkg/microservice/warpdrive/config" "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types/task" + "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" "github.com/koderover/zadig/pkg/tool/kodo" ) @@ -98,7 +104,7 @@ type Preview struct { func ToPreview(sb map[string]interface{}) (*Preview, error) { var pre *Preview if err := IToi(sb, &pre); err != nil { - return nil, fmt.Errorf("convert interface to SubTaskPreview error: %v", err) + return nil, fmt.Errorf("convert interface to SubTaskPreview error: %s", err) } return pre, nil } @@ -106,15 +112,23 @@ func ToPreview(sb map[string]interface{}) (*Preview, error) { func ToBuildTask(sb map[string]interface{}) (*task.Build, error) { var t *task.Build if err := IToi(sb, &t); err != nil { - return nil, fmt.Errorf("convert interface to BuildTaskV2 error: %v", err) + return nil, fmt.Errorf("convert interface to BuildTaskV2 error: %s", err) } return t, nil } -func ToArtifactTask(sb map[string]interface{}) (*task.Artifact, error) { - var t *task.Artifact +func ToScanningTask(sb map[string]interface{}) (*task.Scanning, error) { + var t *task.Scanning if err := IToi(sb, &t); err != nil { - return nil, fmt.Errorf("convert interface to ArtifactTask error: %v", err) + return nil, fmt.Errorf("convert interface to Scanning task error: %s", err) + } + return t, nil +} + +func ToArtifactTask(sb map[string]interface{}) (*task.ArtifactPackage, error) { + var t *task.ArtifactPackage + if err := IToi(sb, &t); err != nil { + return nil, fmt.Errorf("convert interface to ArtifactTask error: %s", err) } return t, nil } @@ -122,7 +136,7 @@ func ToArtifactTask(sb map[string]interface{}) (*task.Artifact, error) { func ToDockerBuildTask(sb map[string]interface{}) (*task.DockerBuild, error) { var t *task.DockerBuild if err := IToi(sb, &t); err != nil { - return nil, fmt.Errorf("convert interface to DockerBuildTask error: %v", err) + return nil, fmt.Errorf("convert interface to DockerBuildTask error: %s", err) } return t, nil } @@ -130,7 +144,7 @@ func ToDockerBuildTask(sb map[string]interface{}) (*task.DockerBuild, error) { func ToDeployTask(sb map[string]interface{}) (*task.Deploy, error) { var t *task.Deploy if err := IToi(sb, &t); err != nil { - return nil, fmt.Errorf("convert interface to DeployTask error: %v", err) + return nil, fmt.Errorf("convert interface to DeployTask error: %s", err) } return t, nil } @@ -138,7 +152,7 @@ func ToDeployTask(sb map[string]interface{}) (*task.Deploy, error) { func ToTestingTask(sb map[string]interface{}) (*task.Testing, error) { var t *task.Testing if err := IToi(sb, &t); err != nil { - return nil, fmt.Errorf("convert interface to Testing error: %v", err) + return nil, fmt.Errorf("convert interface to Testing error: %s", err) } return t, nil } @@ -146,7 +160,7 @@ func ToTestingTask(sb map[string]interface{}) (*task.Testing, error) { func ToDistributeToS3Task(sb map[string]interface{}) (*task.DistributeToS3, error) { var t *task.DistributeToS3 if err := IToi(sb, &t); err != nil { - return nil, fmt.Errorf("convert interface to DistributeToS3Task error: %v", err) + return nil, fmt.Errorf("convert interface to DistributeToS3Task error: %s", err) } return t, nil } @@ -154,15 +168,23 @@ func ToDistributeToS3Task(sb map[string]interface{}) (*task.DistributeToS3, erro func ToReleaseImageTask(sb map[string]interface{}) (*task.ReleaseImage, error) { var t *task.ReleaseImage if err := IToi(sb, &t); err != nil { - return nil, fmt.Errorf("convert interface to ReleaseImageTask error: %v", err) + return nil, fmt.Errorf("convert interface to ReleaseImageTask error: %s", err) } return t, nil } +func ToArtifactPackageTask(sb map[string]interface{}) (*task.ArtifactPackageTaskArgs, error) { + var ret *task.ArtifactPackageTaskArgs + if err := IToi(sb, &ret); err != nil { + return nil, fmt.Errorf("convert interface to ArtifactPackageTaskArgs error: %s", err) + } + return ret, nil +} + func ToJiraTask(sb map[string]interface{}) (*task.Jira, error) { var t *task.Jira if err := IToi(sb, &t); err != nil { - return nil, fmt.Errorf("convert interface to JiraTask error: %v", err) + return nil, fmt.Errorf("convert interface to JiraTask error: %s", err) } return t, nil } @@ -170,7 +192,7 @@ func ToJiraTask(sb map[string]interface{}) (*task.Jira, error) { func ToSecurityTask(sb map[string]interface{}) (*task.Security, error) { var t *task.Security if err := IToi(sb, &t); err != nil { - return nil, fmt.Errorf("convert interface to securityTask error: %v", err) + return nil, fmt.Errorf("convert interface to securityTask error: %s", err) } return t, nil } @@ -178,7 +200,7 @@ func ToSecurityTask(sb map[string]interface{}) (*task.Security, error) { func ToJenkinsBuildTask(sb map[string]interface{}) (*task.JenkinsBuild, error) { var task *task.JenkinsBuild if err := IToi(sb, &task); err != nil { - return nil, fmt.Errorf("convert interface to JenkinsBuildTask error: %v", err) + return nil, fmt.Errorf("convert interface to JenkinsBuildTask error: %s", err) } return task, nil } @@ -186,12 +208,93 @@ func ToJenkinsBuildTask(sb map[string]interface{}) (*task.JenkinsBuild, error) { func IToi(before interface{}, after interface{}) error { b, err := json.Marshal(before) if err != nil { - return fmt.Errorf("marshal task error: %v", err) + return fmt.Errorf("marshal task error: %s", err) } if err := json.Unmarshal(b, &after); err != nil { - return fmt.Errorf("unmarshal task error: %v", err) + return fmt.Errorf("unmarshal task error: %s", err) } return nil } + +func ToTriggerTask(sb map[string]interface{}) (*task.Trigger, error) { + var trigger *task.Trigger + if err := task.IToi(sb, &trigger); err != nil { + return nil, fmt.Errorf("convert interface to triggerTask error: %s", err) + } + return trigger, nil +} + +func ToExtensionTask(sb map[string]interface{}) (*task.Extension, error) { + var extension *task.Extension + if err := task.IToi(sb, &extension); err != nil { + return nil, fmt.Errorf("convert interface to extensionTask error: %s", err) + } + return extension, nil +} + +func GetK8sClients(hubServerAddr, clusterID string) (crClient.Client, kubernetes.Interface, *rest.Config, error) { + controllerRuntimeClient, err := kubeclient.GetKubeClient(hubServerAddr, clusterID) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get controller runtime client: %s", err) + } + + clientset, err := kubeclient.GetKubeClientSet(hubServerAddr, clusterID) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get clientset: %s", err) + } + + restConfig, err := kubeclient.GetRESTConfig(hubServerAddr, clusterID) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get rest config: %s", err) + } + + return controllerRuntimeClient, clientset, restConfig, nil +} + +// InstantiateBuildSysVariables instantiate system variables for build module +func InstantiateBuildSysVariables(jobCtx *task.JobCtx) []*task.KeyVal { + ret := make([]*task.KeyVal, 0) + for index, repo := range jobCtx.Builds { + + repoNameIndex := fmt.Sprintf("REPONAME_%d", index) + ret = append(ret, &task.KeyVal{Key: fmt.Sprintf(repoNameIndex), Value: repo.RepoName, IsCredential: false}) + + repoName := strings.Replace(repo.RepoName, "-", "_", -1) + + repoIndex := fmt.Sprintf("REPO_%d", index) + ret = append(ret, &task.KeyVal{Key: fmt.Sprintf(repoIndex), Value: repoName, IsCredential: false}) + + if len(repo.Branch) > 0 { + ret = append(ret, &task.KeyVal{Key: fmt.Sprintf("%s_BRANCH", repoName), Value: repo.Branch, IsCredential: false}) + } + + if len(repo.Tag) > 0 { + ret = append(ret, &task.KeyVal{Key: fmt.Sprintf("%s_TAG", repoName), Value: repo.Tag, IsCredential: false}) + } + + if repo.PR > 0 { + ret = append(ret, &task.KeyVal{Key: fmt.Sprintf("%s_PR", repoName), Value: strconv.Itoa(repo.PR), IsCredential: false}) + } + + if len(repo.CommitID) > 0 { + ret = append(ret, &task.KeyVal{Key: fmt.Sprintf("%s_COMMIT_ID", repoName), Value: repo.CommitID, IsCredential: false}) + } + } + return ret +} + +// getReaperImage generates the image used to run reaper +// depends on the setting on page 'Build' +func getReaperImage(reaperImage, buildOS, imageFrom string) string { + // for built-in image, reaperImage and buildOs can generate a complete image + // reaperImage: koderover.tencentcloudcr.com/koderover-public/build-base:${BuildOS}-amd64 + // buildOS: focal xenial bionic + jobImage := strings.ReplaceAll(reaperImage, "${BuildOS}", buildOS) + // for custom image, buildOS represents the exact custom image + if imageFrom == setting.ImageFromCustom { + jobImage = buildOS + } + return jobImage +} diff --git a/pkg/microservice/warpdrive/core/service/types/environment.go b/pkg/microservice/warpdrive/core/service/types/environment.go index b8ac9e7eeb6e9734c1229a47204e1a9783190927..bf9a386ab4f855f89dbc10be07e0591716333230 100644 --- a/pkg/microservice/warpdrive/core/service/types/environment.go +++ b/pkg/microservice/warpdrive/core/service/types/environment.go @@ -60,26 +60,19 @@ type RenderKV struct { Services []string `bson:"services" json:"services"` } -type GitRepoConfig struct { - CodehostID int `bson:"codehost_id,omitempty"` - Owner string `bson:"owner,omitempty"` - Repo string `bson:"repo,omitempty"` - Branch string `bson:"branch,omitempty"` -} - -type OverrideYaml struct { - YamlSource string `bson:"yaml_source,omitempty" json:"yaml_source,omitempty"` - YamlContent string `bson:"yaml_content,omitempty" json:"yaml_content,omitempty"` - GitRepoConfig *GitRepoConfig `bson:"git_repo_config,omitempty" json:"git_repo_config,omitempty"` - ValuesPaths []string `bson:"values_paths,omitempty" json:"values_paths,omitempty"` +type CustomYaml struct { + YamlContent string `json:"yaml_content,omitempty"` + Source string `json:"source"` + AutoSync bool `json:"auto_sync"` + SourceDetail interface{} `json:"source_detail"` } type RenderChart struct { - ServiceName string `bson:"service_name,omitempty" json:"service_name,omitempty"` - ChartVersion string `bson:"chart_version,omitempty" json:"chart_version,omitempty"` - ValuesYaml string `bson:"values_yaml,omitempty" json:"values_yaml,omitempty"` - OverrideYaml *OverrideYaml `bson:"override_yaml,omitempty" json:"override_yaml,omitempty"` - OverrideValues string `bson:"override_values,omitempty" json:"override_values,omitempty"` + ServiceName string `json:"service_name,omitempty"` + ChartVersion string `json:"chart_version,omitempty"` + ValuesYaml string `json:"values_yaml,omitempty"` + OverrideYaml *CustomYaml `json:"override_yaml,omitempty"` + OverrideValues string `json:"override_values,omitempty"` } type ProductAuth struct { @@ -98,17 +91,6 @@ type Service struct { EnvConfigs []*EnvConfig `bson:"-" json:"env_configs,omitempty"` } -//type EnvConfig struct { -// EnvName string `json:"env_name"` -// HostIDs []string `json:"host_ids"` -//} -// -//// Container ... -//type Container struct { -// Name string `bson:"name" json:"name"` -// Image string `bson:"image" json:"image"` -//} - // Config ... type Config struct { ConfigName string `bson:"config_name" json:"config_name"` @@ -121,3 +103,13 @@ func (rc *RenderChart) GetOverrideYaml() string { } return rc.OverrideYaml.YamlContent } + +func (p *Product) GetServiceMap() map[string]*Service { + ret := make(map[string]*Service) + for _, group := range p.Services { + for _, svc := range group { + ret[svc.ServiceName] = svc + } + } + return ret +} diff --git a/pkg/microservice/warpdrive/core/service/types/notify.go b/pkg/microservice/warpdrive/core/service/types/notify.go index fb876785f120df5178b189656b5af074c461736a..8b96e9ffe9394512282967e9289fed349716261f 100644 --- a/pkg/microservice/warpdrive/core/service/types/notify.go +++ b/pkg/microservice/warpdrive/core/service/types/notify.go @@ -16,7 +16,10 @@ limitations under the License. package types -import "github.com/koderover/zadig/pkg/microservice/warpdrive/config" +import ( + "github.com/koderover/zadig/pkg/microservice/warpdrive/config" + "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/common" +) //// AnnouncementCtx ... //type AnnouncementCtx struct { @@ -35,6 +38,7 @@ type PipelineStatusCtx struct { Type config.PipelineType `bson:"type" json:"type"` Status config.Status `bson:"status" json:"status,omitempty"` TeamName string `bson:"team" json:"team"` + Stages []*common.Stage `bson:"stages" json:"stages"` } // diff --git a/pkg/microservice/warpdrive/core/service/types/packager.go b/pkg/microservice/warpdrive/core/service/types/packager.go new file mode 100644 index 0000000000000000000000000000000000000000..19aac50ae5cf4003e2e114c85effc29bee11c7d2 --- /dev/null +++ b/pkg/microservice/warpdrive/core/service/types/packager.go @@ -0,0 +1,27 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types/task" + +type ArtifactPackagerContext struct { + JobType string `yaml:"job_type"` + ProgressFile string `yaml:"progress_file"` + Images []*task.ImagesByService `yaml:"images"` + SourceRegistries []*DockerRegistry `yaml:"source_registries"` + TargetRegistries []*DockerRegistry `yaml:"target_registries"` +} diff --git a/pkg/microservice/warpdrive/core/service/types/predator.go b/pkg/microservice/warpdrive/core/service/types/predator.go index ccb37a187f11cf5f38af348183ff081c72273e5d..c866e19c1611fca0ab12407a741e8848b8d7c1ab 100644 --- a/pkg/microservice/warpdrive/core/service/types/predator.go +++ b/pkg/microservice/warpdrive/core/service/types/predator.go @@ -19,9 +19,10 @@ package types import "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types/task" type PredatorContext struct { - JobType string `yaml:"job_type"` - DockerRegistry *DockerRegistry `yaml:"docker_registry"` - DockerBuildCtx *task.DockerBuildCtx `yaml:"build_ctx"` - OnSetup string `yaml:"setup,omitempty"` - ReleaseImages []task.RepoImage `yaml:"release_images"` + JobType string `yaml:"job_type"` + DockerRegistry *DockerRegistry `yaml:"docker_registry"` + DockerBuildCtx *task.DockerBuildCtx `yaml:"build_ctx"` + OnSetup string `yaml:"setup,omitempty"` + ReleaseImages []task.RepoImage `yaml:"release_images"` + DistributeInfo []*task.DistributeInfo `yaml:"distribute_info"` } diff --git a/pkg/microservice/warpdrive/core/service/types/reaper.go b/pkg/microservice/warpdrive/core/service/types/reaper.go index 8ac5c01c37420a241b91c2ae4ce8d9772cda327d..240b2996aac90241de6b37c0d3cc4bdccd4ecae9 100644 --- a/pkg/microservice/warpdrive/core/service/types/reaper.go +++ b/pkg/microservice/warpdrive/core/service/types/reaper.go @@ -23,13 +23,17 @@ import ( "github.com/koderover/zadig/pkg/microservice/warpdrive/config" "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types/task" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/types" ) // Context ... type Context struct { + // API token 服务访问使用的api token + APIToken string `yaml:"api_token"` // Workspace 容器工作目录 [必填] Workspace string `yaml:"workspace"` + // TODO: Deprecated. // CleanWorkspace 是否清理工作目录 [选填, 默认为 false] CleanWorkspace bool `yaml:"clean_workspace"` @@ -105,11 +109,39 @@ type Context struct { // IgnoreCache ignore docker build cache [runtime] IgnoreCache bool `yaml:"ignore_cache"` - StorageEndpoint string `yaml:"storage_endpoint"` - StorageAK string `yaml:"storage_ak"` - StorageSK string `yaml:"storage_sk"` - StorageBucket string `yaml:"storage_bucket"` - StorageProvider int8 `yaml:"storage_provider"` + StorageEndpoint string `yaml:"storage_endpoint"` + StorageAK string `yaml:"storage_ak"` + StorageSK string `yaml:"storage_sk"` + StorageBucket string `yaml:"storage_bucket"` + StorageProvider int8 `yaml:"storage_provider"` + ArtifactInfo *ArtifactInfo `yaml:"artifact_info"` + ArtifactPath string `yaml:"artifact_path"` + AesKey string `yaml:"aes_key"` + + // New since V1.10.0. + CacheEnable bool `yaml:"cache_enable"` + Cache types.Cache `yaml:"cache"` + CacheDirType types.CacheDirType `yaml:"cache_dir_type"` + CacheUserDir string `yaml:"cache_user_dir"` + + // Upload To S3 related context + UploadEnabled bool `yaml:"upload_enabled"` + UploadStorageInfo *types.ObjectStorageInfo `yaml:"upload_storage_info"` + UploadInfo []*types.ObjectStoragePathDetail `yaml:"upload_info"` + + // scanner used flag + ScannerFlag bool `yaml:"scanner_flag"` + ScannerType string `yaml:"scanner_type"` + SonarParameter string `yaml:"sonar_parameter"` + SonarServer string `yaml:"sonar_server"` + SonarLogin string `yaml:"sonar_login"` +} + +type ArtifactInfo struct { + URL string `yaml:"url"` + WorkflowName string `yaml:"workflow_name"` + TaskID int64 `yaml:"task_id"` + FileName string `yaml:"file_name"` } // Proxy 翻墙配置信息 @@ -185,20 +217,25 @@ type Install struct { // Repo ... type Repo struct { - Source string `yaml:"source"` - Address string `yaml:"address"` - Owner string `yaml:"owner"` - Name string `yaml:"name"` - RemoteName string `yaml:"remote_name"` - Branch string `yaml:"branch"` - PR int `yaml:"pr"` - Tag string `yaml:"tag"` - CheckoutPath string `yaml:"checkout_path"` - SubModules bool `yaml:"submodules"` - OauthToken string `yaml:"oauthToken"` - User string `yaml:"username"` - Password string `yaml:"password"` - CheckoutRef string `yaml:"checkout_ref"` + Source string `yaml:"source"` + Address string `yaml:"address"` + Owner string `yaml:"owner"` + Namespace string `yaml:"namespace"` + Name string `yaml:"name"` + RemoteName string `yaml:"remote_name"` + Branch string `yaml:"branch"` + PR int `yaml:"pr"` + Tag string `yaml:"tag"` + CheckoutPath string `yaml:"checkout_path"` + SubModules bool `yaml:"submodules"` + OauthToken string `yaml:"oauthToken"` + User string `yaml:"username"` + Password string `yaml:"password"` + CheckoutRef string `yaml:"checkout_ref"` + EnableProxy bool `yaml:"enable_proxy"` + AuthType types.AuthType `yaml:"auth_type,omitempty"` + SSHKey string `yaml:"ssh_key,omitempty"` + PrivateAccessToken string `yaml:"private_access_token,omitempty"` } // PRRef returns refs format @@ -259,10 +296,11 @@ type GinkgoTest struct { // DockerRegistry 推送镜像到 docker registry 配置 type DockerRegistry struct { - Host string `yaml:"host"` - Namespace string `yaml:"namespace"` - UserName string `yaml:"username"` - Password string `yaml:"password"` + RegistryID string `yaml:"registry_id"` + Host string `yaml:"host"` + Namespace string `yaml:"namespace"` + UserName string `yaml:"username"` + Password string `yaml:"password"` } // Git ... diff --git a/pkg/microservice/warpdrive/core/service/types/renderset.go b/pkg/microservice/warpdrive/core/service/types/renderset.go index 0552cad503d5863988b16af60f3cba72630785db..250b89ab03ca82dfc8173044ca147541945a6ee1 100644 --- a/pkg/microservice/warpdrive/core/service/types/renderset.go +++ b/pkg/microservice/warpdrive/core/service/types/renderset.go @@ -22,20 +22,14 @@ type RenderSet struct { Name string `bson:"name" json:"name"` Revision int64 `bson:"revision" json:"revision"` // 可以为空，空时为产品模板默认的渲染集，非空时为环境的渲染集 - EnvName string `bson:"env_name,omitempty" json:"env_name,omitempty"` - ProductTmpl string `bson:"product_tmpl" json:"product_tmpl"` - Team string `bson:"team,omitempty" json:"team,omitempty"` - UpdateTime int64 `bson:"update_time" json:"update_time"` - UpdateBy string `bson:"update_by" json:"update_by"` - IsDefault bool `bson:"is_default" json:"is_default"` // 是否是默认配置 - KVs []*RenderKV `bson:"kvs,omitempty" json:"kvs,omitempty"` - ChartInfos []*RenderChart `bson:"chart_infos,omitempty" json:"chart_infos,omitempty"` - Description string `bson:"description,omitempty" json:"description,omitempty"` + EnvName string `bson:"env_name,omitempty" json:"env_name,omitempty"` + ProductTmpl string `bson:"product_tmpl" json:"product_tmpl"` + Team string `bson:"team,omitempty" json:"team,omitempty"` + UpdateTime int64 `bson:"update_time" json:"update_time"` + UpdateBy string `bson:"update_by" json:"update_by"` + IsDefault bool `bson:"is_default" json:"is_default"` // 是否是默认配置 + DefaultValues string `bson:"default_values,omitempty" json:"default_values,omitempty"` //环境默认变量 ›yaml content + KVs []*RenderKV `bson:"kvs,omitempty" json:"kvs,omitempty"` + ChartInfos []*RenderChart `bson:"chart_infos,omitempty" json:"chart_infos,omitempty"` + Description string `bson:"description,omitempty" json:"description,omitempty"` } - -//// RenderChart ... -//type RenderChart struct { -// ServiceName string `bson:"service_name,omitempty" json:"service_name,omitempty"` -// ChartVersion string `bson:"chart_version,omitempty" json:"chart_version,omitempty"` -// ValuesYaml string `bson:"values_yaml,omitempty" json:"values_yaml,omitempty"` -//} diff --git a/pkg/microservice/warpdrive/core/service/types/task/artifact_package.go b/pkg/microservice/warpdrive/core/service/types/task/artifact_package.go new file mode 100644 index 0000000000000000000000000000000000000000..5ebe24fb0eea78b73cdd0bfdc46df29f65640f25 --- /dev/null +++ b/pkg/microservice/warpdrive/core/service/types/task/artifact_package.go @@ -0,0 +1,50 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package task + +import ( + "fmt" + + "github.com/koderover/zadig/pkg/microservice/warpdrive/config" +) + +type ArtifactPackage struct { + TaskType config.TaskType `bson:"type" json:"type"` + Enabled bool `bson:"enabled" json:"enabled"` + TaskStatus config.Status `bson:"status" json:"status"` + Progress string `bson:"progress" json:"progress"` + Timeout int `bson:"timeout,omitempty" json:"timeout,omitempty"` + Error string `bson:"error,omitempty" json:"error,omitempty"` + StartTime int64 `bson:"start_time,omitempty" json:"start_time,omitempty"` + EndTime int64 `bson:"end_time,omitempty" json:"end_time,omitempty"` + LogFile string `bson:"log_file" json:"log_file"` + + // source images + Images []*ImagesByService `bson:"images" json:"images"` + // target registries to push images + SourceRegistries []string `bson:"source_registries" json:"source_registries"` + // target registries to push images + TargetRegistries []string `bson:"target_registries" json:"target_registries"` +} + +func (ri *ArtifactPackage) ToSubTask() (map[string]interface{}, error) { + var task map[string]interface{} + if err := IToi(ri, &task); err != nil { + return nil, fmt.Errorf("convert ReleaseImageTask to interface error: %v", err) + } + return task, nil +} diff --git a/pkg/microservice/warpdrive/core/service/types/task/build.go b/pkg/microservice/warpdrive/core/service/types/task/build.go index eed15c975d892d25a8fd830f54458d72e0d12e0c..80130bc01cc520fa4153147fb84af191c5ce04ae 100644 --- a/pkg/microservice/warpdrive/core/service/types/task/build.go +++ b/pkg/microservice/warpdrive/core/service/types/task/build.go @@ -21,12 +21,14 @@ import ( "github.com/koderover/zadig/pkg/microservice/warpdrive/config" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/types" ) type Build struct { - TaskType config.TaskType `bson:"type" json:"type"` - Enabled bool `bson:"enabled" json:"enabled"` - TaskStatus config.Status `bson:"status" json:"status"` + TaskType config.TaskType `bson:"type" json:"type"` + Enabled bool `bson:"enabled" json:"enabled"` + TaskStatus config.Status `bson:"status" json:"status"` + ProductName string `bson:"product_name" json:"product_name"` // 新增一个service表示服务名称 Service string `bson:"service" json:"service"` // 该名称实际为服务组件名称 @@ -45,16 +47,33 @@ type Build struct { ImageFrom string `bson:"image_from" json:"image_from,omitempty"` ImageID string `bson:"image_id" json:"image_id"` ResReq setting.Request `bson:"res_req" json:"res_req"` + ResReqSpec setting.RequestSpec `bson:"res_req_spec" json:"res_req_spec"` LogFile string `bson:"log_file" json:"log_file"` InstallCtx []*Install `bson:"-" json:"install_ctx,omitempty"` - Registries []*RegistryNamespace `bson:"-" json:"registries"` + Registries []*RegistryNamespace `bson:"-" json:"registries"` StaticCheckStatus *StaticCheckStatus `bson:"static_check_status,omitempty" json:"static_check_status,omitempty"` UTStatus *UTStatus `bson:"ut_status,omitempty" json:"ut_status,omitempty"` DockerBuildStatus *DockerBuildStatus `bson:"docker_build_status,omitempty" json:"docker_build_status,omitempty"` BuildStatus *BuildStatus `bson:"build_status,omitempty" json:"build_status,omitempty"` IsRestart bool `bson:"is_restart" json:"is_restart"` // Get the host bound to the environment of the cloud host service configuration - EnvHostInfo map[string][]string `bson:"env_host_info,omitempty" json:"env_host_info,omitempty"` + EnvHostInfo map[string][]string `bson:"env_host_info,omitempty" json:"env_host_info,omitempty"` + EnvHostNames map[string][]string `bson:"env_host_names,omitempty" json:"env_host_names,omitempty"` + ArtifactInfo *ArtifactInfo `bson:"artifact_info,omitempty" json:"artifact_info,omitempty"` + ClusterID string `bson:"cluster_id,omitempty" json:"cluster_id,omitempty"` + + // New since V1.10.0. + Cache types.Cache `bson:"cache" json:"cache"` + CacheEnable bool `bson:"cache_enable" json:"cache_enable"` + CacheDirType types.CacheDirType `bson:"cache_dir_type" json:"cache_dir_type"` + CacheUserDir string `bson:"cache_user_dir" json:"cache_user_dir"` +} + +type ArtifactInfo struct { + URL string `bson:"url" json:"url"` + WorkflowName string `bson:"workflow_name" json:"workflow_name"` + TaskID int64 `bson:"task_id" json:"task_id"` + FileName string `bson:"file_name" json:"file_name"` } type Item struct { @@ -76,7 +95,7 @@ type Install struct { } type RegistryNamespace struct { - OrgID int `bson:"org_id" json:"org_id"` + ID string `bson:"_id,omitempty" json:"id,omitempty"` RegAddr string `bson:"reg_addr" json:"reg_addr"` RegType string `bson:"reg_type" json:"reg_type"` RegProvider string `bson:"reg_provider" json:"reg_provider"` @@ -137,9 +156,11 @@ type DockerBuildStatus struct { } type JobCtx struct { - EnableProxy bool `bson:"enable_proxy" json:"enable_proxy"` - Proxy *Proxy `bson:"proxy" json:"proxy"` - CleanWorkspace bool `bson:"clean_workspace" json:"clean_workspace"` + EnableProxy bool `bson:"enable_proxy" json:"enable_proxy"` + Proxy *Proxy `bson:"proxy" json:"proxy"` + + // TODO: Deprecated. + CleanWorkspace bool `bson:"clean_workspace" json:"clean_workspace"` // BuildJobCtx Builds []*Repository `bson:"builds" json:"builds"` @@ -162,8 +183,12 @@ type JobCtx struct { FileArchiveCtx *FileArchiveCtx `bson:"file_archive_ctx,omitempty" json:"file_archive_ctx,omitempty"` // TestType TestType string `bson:"test_type" json:"test_type"` - // Caches - Caches []string `bson:"caches" json:"caches"` + + // TODO: Deprecated. + Caches []string `bson:"caches" json:"caches"` + + // buildV3 + ArtifactPath string `bson:"artifact_path,omitempty" json:"artifact_path,omitempty"` ArtifactPaths []string `bson:"artifact_paths,omitempty" json:"artifact_paths,omitempty"` IsHasArtifact bool `bson:"is_has_artifact" json:"is_has_artifact"` // StorageUri is used for qbox release-candidates @@ -173,6 +198,11 @@ type JobCtx struct { ClassicBuild bool `bson:"classic_build" json:"classic_build"` PostScripts string `bson:"post_scripts,omitempty" json:"post_scripts"` PMDeployScripts string `bson:"pm_deploy_scripts,omitempty" json:"pm_deploy_scripts"` + + // Upload To S3 related context + UploadEnabled bool `json:"upload_enabled"` + UploadStorageInfo *types.ObjectStorageInfo `json:"upload_storage_info"` + UploadInfo []*types.ObjectStoragePathDetail `json:"upload_info"` } type SSH struct { @@ -180,6 +210,7 @@ type SSH struct { Name string `json:"name"` UserName string `json:"user_name"` IP string `json:"ip"` + Port int64 `json:"port"` IsProd bool `json:"is_prod"` Label string `json:"label"` PrivateKey string `json:"private_key"` @@ -191,11 +222,13 @@ type SSH struct { // DockerFile: dockerfile名称, 默认为Dockerfile // ImageBuild: build image镜像全称, e.g. xxx.com/release-candidates/image:tag type DockerBuildCtx struct { - WorkDir string `yaml:"work_dir" bson:"work_dir" json:"work_dir"` - DockerFile string `yaml:"docker_file" bson:"docker_file" json:"docker_file"` - ImageName string `yaml:"image_name" bson:"image_name" json:"image_name"` - BuildArgs string `yaml:"build_args" bson:"build_args" json:"build_args"` - ImageReleaseTag string `yaml:"image_release_tag,omitempty" bson:"image_release_tag,omitempty" json:"image_release_tag"` + WorkDir string `yaml:"work_dir" bson:"work_dir" json:"work_dir"` + DockerFile string `yaml:"docker_file" bson:"docker_file" json:"docker_file"` + ImageName string `yaml:"image_name" bson:"image_name" json:"image_name"` + BuildArgs string `yaml:"build_args" bson:"build_args" json:"build_args"` + ImageReleaseTag string `yaml:"image_release_tag,omitempty" bson:"image_release_tag,omitempty" json:"image_release_tag"` + Source string `yaml:"source" bson:"source" json:"source"` + DockerTemplateContent string `yaml:"docker_template_content" bson:"docker_template_content" json:"docker_template_content"` } type FileArchiveCtx struct { @@ -212,10 +245,19 @@ func (buildCtx *DockerBuildCtx) GetDockerFile() string { return buildCtx.DockerFile } +type ParameterSettingType string + +const ( + StringType ParameterSettingType = "string" + ChoiceType ParameterSettingType = "choice" +) + type KeyVal struct { - Key string `bson:"key" json:"key"` - Value string `bson:"value" json:"value"` - IsCredential bool `bson:"is_credential" json:"is_credential"` + Key string `bson:"key" json:"key"` + Value string `bson:"value" json:"value"` + Type ParameterSettingType `bson:"type,omitempty" json:"type,omitempty"` + ChoiceOption []string `bson:"choice_option,omitempty" json:"choice_option,omitempty"` + IsCredential bool `bson:"is_credential" json:"is_credential"` } type Repository struct { @@ -223,6 +265,7 @@ type Repository struct { Source string `bson:"source,omitempty" json:"source,omitempty"` RepoOwner string `bson:"repo_owner" json:"repo_owner"` RepoName string `bson:"repo_name" json:"repo_name"` + RepoNamespace string `bson:"repo_namespace" json:"repo_namespace"` RemoteName string `bson:"remote_name,omitempty" json:"remote_name,omitempty"` Branch string `bson:"branch" json:"branch"` PR int `bson:"pr,omitempty" json:"pr,omitempty"` @@ -246,6 +289,11 @@ type Repository struct { ProjectUUID string `bson:"project_uuid,omitempty" json:"project_uuid,omitempty"` RepoUUID string `bson:"repo_uuid,omitempty" json:"repo_uuid,omitempty"` RepoID string `bson:"repo_id,omitempty" json:"repo_id,omitempty"` + EnableProxy bool `bson:"enable_proxy,omitempty" json:"enable_proxy,omitempty"` + // The address of the code base input of the other type + AuthType types.AuthType `bson:"auth_type,omitempty" json:"auth_type,omitempty"` + SSHKey string `bson:"ssh_key,omitempty" json:"ssh_key,omitempty"` + PrivateAccessToken string `bson:"private_access_token,omitempty" json:"private_access_token,omitempty"` } type BuildStep struct { diff --git a/pkg/microservice/warpdrive/core/service/types/task/config_payload.go b/pkg/microservice/warpdrive/core/service/types/task/config_payload.go index 2692dbc316c8e8254819b37c2b55eaf10414708b..eac009dbf6232033f065fc909e951cc6544ec316 100644 --- a/pkg/microservice/warpdrive/core/service/types/task/config_payload.go +++ b/pkg/microservice/warpdrive/core/service/types/task/config_payload.go @@ -18,9 +18,12 @@ package task import ( "fmt" + + corev1 "k8s.io/api/core/v1" ) type ConfigPayload struct { + APIToken string `json:"api_token"` Proxy Proxy `json:"proxy"` S3Storage S3Config `json:"s3_storage"` Github GithubConfig `json:"github"` @@ -39,6 +42,7 @@ type ConfigPayload struct { CustomDNSSupported bool `json:"custom_dns_supported"` HubServerAddr string DeployClusterID string + AesKey string `json:"aes_key"` RepoConfigs map[string]*RegistryNamespace @@ -48,6 +52,12 @@ type ConfigPayload struct { // ResetCache means ignore workspace cache ResetCache bool `json:"reset_cache"` PrivateKeys []*PrivateKey `json:"private_keys"` + K8SClusters []*K8SCluster `json:"k8s_clusters"` + + RegistryID string `json:"registry_id"` + + // build concurrency settings + BuildConcurrency int64 `json:"build_concurrency"` } func (cp *ConfigPayload) GetGitKnownHost() string { @@ -81,15 +91,15 @@ type S3Config struct { type GithubConfig struct { // github API access token - AccessToken string + AccessToken string `json:"access_token"` // github ssh key with base64 encoded - SSHKey string + SSHKey string `json:"ssh_key"` // github knownhost - KnownHost string + KnownHost string `json:"known_host"` // github app private key - AppKey string + AppKey string `json:"app_key"` // gihhub app id - AppID int + AppID int `json:"app_id"` } type GitlabConfig struct { @@ -120,14 +130,17 @@ type RegistryConfig struct { type ReleaseConfig struct { // ReaperImage sets build job image - // e.g. xxx.com/poetry-resources/reaper-plugin:1.0.0 + // e.g. xxx.com/resources/reaper-plugin:1.0.0 ReaperImage string // ReaperBinaryFile sets download url of reaper binary file in build job // e.g. http://resource.koderover.com/reaper-20201014203000 ReaperBinaryFile string // PredatorImage sets docker build image - // e.g. xxx.com/poetry-resources/predator-plugin:v0.1.0 + // e.g. xxx.com/resources/predator-plugin:v0.1.0 PredatorImage string + // PackagerImage sets docker build image + // e.g. xxx.com/resources/packager-plugin:v0.1.0 + PackagerImage string } type ImageReleaseConfig struct { @@ -147,3 +160,20 @@ type JenkinsBuildConfig struct { type PrivateKey struct { Name string `json:"name"` } + +type K8SCluster struct { + ID string `json:"id,omitempty" bson:"id,omitempty"` + Name string `json:"name" bson:"name"` + AdvancedConfig *AdvancedConfig `json:"advanced_config,omitempty" bson:"advanced_config,omitempty"` +} + +type AdvancedConfig struct { + Strategy string `json:"strategy,omitempty" bson:"strategy,omitempty"` + NodeLabels []*NodeSelectorRequirement `json:"node_labels,omitempty" bson:"node_labels,omitempty"` +} + +type NodeSelectorRequirement struct { + Key string `json:"key"` + Value []string `json:"value"` + Operator corev1.NodeSelectorOperator `json:"operator"` +} diff --git a/pkg/microservice/warpdrive/core/service/types/task/deploy.go b/pkg/microservice/warpdrive/core/service/types/task/deploy.go index 2014e12d39598528b4c31ec4872b4b532f83b4b5..128427208e43172bc63ba749cb2339c5c25248e6 100644 --- a/pkg/microservice/warpdrive/core/service/types/task/deploy.go +++ b/pkg/microservice/warpdrive/core/service/types/task/deploy.go @@ -20,6 +20,7 @@ import ( "fmt" "github.com/koderover/zadig/pkg/microservice/warpdrive/config" + "github.com/koderover/zadig/pkg/setting" ) type Resource struct { @@ -31,26 +32,28 @@ type Resource struct { // Deploy 容器部署任务 type Deploy struct { - TaskType config.TaskType `bson:"type" json:"type"` - Enabled bool `bson:"enabled" json:"enabled"` - TaskStatus config.Status `bson:"status" json:"status"` - Namespace string `bson:"namespace" json:"namespace"` - EnvName string `bson:"env_name" json:"env_name"` - ProductName string `bson:"product_name" json:"product_name"` - ServiceName string `bson:"service_name" json:"service_name"` - ServiceType string `bson:"service_type" json:"service_type"` - ServiceRevision int64 `bson:"service_revision,omitempty" json:"service_revision,omitempty"` - ContainerName string `bson:"container_name" json:"container_name"` - Image string `bson:"image" json:"image"` - Timeout int `bson:"timeout,omitempty" json:"timeout,omitempty"` - Error string `bson:"error,omitempty" json:"error,omitempty"` - StartTime int64 `bson:"start_time,omitempty" json:"start_time,omitempty"` - EndTime int64 `bson:"end_time,omitempty" json:"end_time,omitempty"` - ClusterID string `bson:"cluster_id,omitempty" json:"cluster_id,omitempty"` - ReplaceResources []Resource `bson:"replace_resources" json:"replace_resources"` - SkipWaiting bool `bson:"skipWaiting" json:"skipWaiting"` - IsRestart bool `bson:"is_restart" json:"is_restart"` - ResetImage bool `bson:"reset_image" json:"reset_image"` + TaskType config.TaskType `bson:"type" json:"type"` + Enabled bool `bson:"enabled" json:"enabled"` + TaskStatus config.Status `bson:"status" json:"status"` + Namespace string `bson:"namespace" json:"namespace"` + EnvName string `bson:"env_name" json:"env_name"` + ProductName string `bson:"product_name" json:"product_name"` + ServiceName string `bson:"service_name" json:"service_name"` + ServiceType string `bson:"service_type" json:"service_type"` + ServiceRevision int64 `bson:"service_revision,omitempty" json:"service_revision,omitempty"` + ReleaseName string `bson:"release_name" json:"release_name"` + ContainerName string `bson:"container_name" json:"container_name"` + Image string `bson:"image" json:"image"` + Timeout int `bson:"timeout,omitempty" json:"timeout,omitempty"` + Error string `bson:"error,omitempty" json:"error,omitempty"` + StartTime int64 `bson:"start_time,omitempty" json:"start_time,omitempty"` + EndTime int64 `bson:"end_time,omitempty" json:"end_time,omitempty"` + ClusterID string `bson:"cluster_id,omitempty" json:"cluster_id,omitempty"` + ReplaceResources []Resource `bson:"replace_resources" json:"replace_resources"` + SkipWaiting bool `bson:"skipWaiting" json:"skipWaiting"` + IsRestart bool `bson:"is_restart" json:"is_restart"` + ResetImage bool `bson:"reset_image" json:"reset_image"` + ResetImagePolicy setting.ResetImagePolicyType `bson:"reset_image_policy" json:"reset_image_policy"` } // SetNamespace ... diff --git a/pkg/microservice/warpdrive/core/service/types/task/docker_build.go b/pkg/microservice/warpdrive/core/service/types/task/docker_build.go index 88eb7dcbc41b244eeea432022e7037d9940b2898..f1e97a6e0ff9e0a450fa5dbe46d8a0dfec8ce737 100644 --- a/pkg/microservice/warpdrive/core/service/types/task/docker_build.go +++ b/pkg/microservice/warpdrive/core/service/types/task/docker_build.go @@ -23,6 +23,8 @@ import ( ) type DockerBuild struct { + Source string `bson:"source" json:"source"` // Dockerfile source + TemplateID string `bson:"template_id" json:"template_id"` TaskType config.TaskType `bson:"type" json:"type"` Enabled bool `bson:"enabled" json:"enabled"` TaskStatus config.Status `bson:"status" json:"status"` diff --git a/pkg/microservice/warpdrive/core/service/types/task/extension.go b/pkg/microservice/warpdrive/core/service/types/task/extension.go new file mode 100644 index 0000000000000000000000000000000000000000..f4eaa1791b012c608a2857f75f1ea8c1d0a660e7 --- /dev/null +++ b/pkg/microservice/warpdrive/core/service/types/task/extension.go @@ -0,0 +1,58 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package task + +import ( + "fmt" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/warpdrive/config" +) + +type Extension struct { + TaskType config.TaskType `bson:"type" json:"type"` + Enabled bool `bson:"enabled" json:"enabled"` + TaskStatus config.Status `bson:"status" json:"status"` + URL string `bson:"url" json:"url"` + Path string `bson:"path,omitempty" json:"path,omitempty"` + IsCallback bool `bson:"is_callback" json:"is_callback"` + Headers []*models.KeyVal `bson:"headers,omitempty" json:"headers"` + Payload string `bson:"payload" json:"payload"` + ResponseBody *string `bson:"response_body" json:"response_body"` + ResponseCode *int `bson:"response_code" json:"response_code"` + ServiceInfos []*ServiceInfo `bson:"service_infos" json:"service_infos"` + Timeout int `bson:"timeout" json:"timeout,omitempty"` + IsRestart bool `bson:"is_restart" json:"is_restart"` + Error string `bson:"error,omitempty" json:"error,omitempty"` + StartTime int64 `bson:"start_time" json:"start_time,omitempty"` + EndTime int64 `bson:"end_time" json:"end_time,omitempty"` +} + +type ServiceInfo struct { + ServiceName string `bson:"service_name" json:"service_name"` + ServiceModule string `bson:"service_module" json:"service_module"` + Image string `bson:"image" json:"image"` +} + +// ToSubTask ... +func (t *Extension) ToSubTask() (map[string]interface{}, error) { + var task map[string]interface{} + if err := IToi(t, &task); err != nil { + return nil, fmt.Errorf("convert extension to interface error: %s", err) + } + return task, nil +} diff --git a/pkg/microservice/warpdrive/core/service/types/task/jenkins.go b/pkg/microservice/warpdrive/core/service/types/task/jenkins.go index 5f6bd942d575e56d36d65542c9ff7862df414d3b..fb4e00c1846e9f6fa3068459b5cb45bfc1be7a66 100644 --- a/pkg/microservice/warpdrive/core/service/types/task/jenkins.go +++ b/pkg/microservice/warpdrive/core/service/types/task/jenkins.go @@ -41,22 +41,24 @@ type JenkinsBuildParam struct { // JenkinsBuild ... type JenkinsBuild struct { - TaskType config.TaskType `bson:"type" json:"type"` - Enabled bool `bson:"enabled" json:"enabled"` - TaskStatus config.Status `bson:"status" json:"status"` - ServiceName string `bson:"service_name" json:"service_name"` - Service string `bson:"service" json:"service"` - OnSetup string `bson:"setup,omitempty" json:"setup,omitempty"` - Timeout int `bson:"timeout" json:"timeout,omitempty"` - Error string `bson:"error" json:"error,omitempty"` - ResReq setting.Request `bson:"res_req" json:"res_req"` - JenkinsIntegration *JenkinsIntegration `bson:"jenkins_integration" json:"jenkins_integration"` - JenkinsBuildArgs *JenkinsBuildArgs `bson:"jenkins_build_args" json:"jenkins_build_args"` - StartTime int64 `bson:"start_time" json:"start_time,omitempty"` - EndTime int64 `bson:"end_time" json:"end_time,omitempty"` - LogFile string `bson:"log_file" json:"log_file"` - Image string `bson:"image,omitempty" json:"image,omitempty"` - IsRestart bool `bson:"is_restart" json:"is_restart"` + TaskType config.TaskType `bson:"type" json:"type"` + Enabled bool `bson:"enabled" json:"enabled"` + TaskStatus config.Status `bson:"status" json:"status"` + ServiceName string `bson:"service_name" json:"service_name"` + Service string `bson:"service" json:"service"` + OnSetup string `bson:"setup,omitempty" json:"setup,omitempty"` + Timeout int `bson:"timeout,omitempty" json:"timeout,omitempty"` + Error string `bson:"error,omitempty" json:"error,omitempty"` + ResReq setting.Request `bson:"res_req" json:"res_req"` + ResReqSpec setting.RequestSpec `bson:"res_req_spec" json:"res_req_spec"` + JenkinsIntegration *JenkinsIntegration `bson:"jenkins_integration" json:"jenkins_integration"` + JenkinsBuildArgs *JenkinsBuildArgs `bson:"jenkins_build_args" json:"jenkins_build_args"` + Registries []*RegistryNamespace `bson:"registries" json:"registries"` + StartTime int64 `bson:"start_time,omitempty" json:"start_time,omitempty"` + EndTime int64 `bson:"end_time,omitempty" json:"end_time,omitempty"` + LogFile string `bson:"log_file" json:"log_file"` + Image string `bson:"image,omitempty" json:"image,omitempty"` + IsRestart bool `bson:"is_restart" json:"is_restart"` } func (j *JenkinsBuild) ToSubTask() (map[string]interface{}, error) { diff --git a/pkg/microservice/warpdrive/core/service/types/task/model.go b/pkg/microservice/warpdrive/core/service/types/task/model.go index 8fc7400a949a076d0355dd3b15654f710facbb4e..fbd489352313eff4edc4245f0b478f1ffb2ece7d 100644 --- a/pkg/microservice/warpdrive/core/service/types/task/model.go +++ b/pkg/microservice/warpdrive/core/service/types/task/model.go @@ -22,6 +22,8 @@ import ( "sync" "github.com/koderover/zadig/pkg/microservice/warpdrive/config" + "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/common" + "github.com/koderover/zadig/pkg/setting" ) type Task struct { @@ -38,7 +40,7 @@ type Task struct { StartTime int64 `bson:"start_time" json:"start_time,omitempty"` EndTime int64 `bson:"end_time" json:"end_time,omitempty"` SubTasks []map[string]interface{} `bson:"sub_tasks" json:"sub_tasks"` - Stages []*Stage `bson:"stages" json:"stages"` + Stages []*common.Stage `bson:"stages" json:"stages"` ReqID string `bson:"req_id,omitempty" json:"req_id,omitempty"` AgentHost string `bson:"agent_host,omitempty" json:"agent_host,omitempty"` DockerHost string `bson:"-" json:"docker_host,omitempty"` @@ -47,43 +49,41 @@ type Task struct { IsDeleted bool `bson:"is_deleted" json:"is_deleted"` IsArchived bool `bson:"is_archived" json:"is_archived"` AgentID string `bson:"agent_id" json:"agent_id"` - // 是否允许同时运行多次 + // is allowed to run multiple times MultiRun bool `bson:"multi_run" json:"multi_run"` - // target 服务名称, k8s为容器名称, 物理机为服务名 + // target is container name when k8s, service name when pm Target string `bson:"target,omitempty" json:"target"` - // 使用预定义编译管理模块中的内容生成SubTasks, - // 查询条件为 服务模板名称: ServiceTmpl, 版本: BuildModuleVer - // 如果为空，则使用pipeline自定义SubTasks + // generate SubTasks with predefine build module, + // query filter param: ServiceTmpl, BuildModuleVer + // if nil，use pipeline self define SubTasks BuildModuleVer string `bson:"build_module_ver,omitempty" json:"build_module_ver"` ServiceName string `bson:"service_name,omitempty" json:"service_name,omitempty"` - // TaskArgs 单服务工作流任务参数 + // TaskArgs single workflow args TaskArgs *TaskArgs `bson:"task_args,omitempty" json:"task_args,omitempty"` - // WorkflowArgs 多服务工作流任务参数 + // WorkflowArgs multi workflow args WorkflowArgs *WorkflowTaskArgs `bson:"workflow_args" json:"workflow_args,omitempty"` - // TestArgs 测试任务参数 + // TestArgs test workflow args TestArgs *TestTaskArgs `bson:"test_args,omitempty" json:"test_args,omitempty"` - // ServiceTaskArgs 脚本部署工作流任务参数 + // ServiceTaskArgs sh deploy args ServiceTaskArgs *ServiceTaskArgs `bson:"service_args,omitempty" json:"service_args,omitempty"` - // ConfigPayload 系统配置信息 - ConfigPayload *ConfigPayload `json:"config_payload,omitempty"` - Error string `bson:"error,omitempty" json:"error,omitempty"` - // OrgID 单租户ID - OrgID int `bson:"org_id,omitempty" json:"org_id,omitempty"` - Services [][]*ProductService `bson:"services" json:"services"` - Render *RenderInfo `bson:"render" json:"render"` - StorageURI string `bson:"storage_uri,omitempty" json:"storage_uri,omitempty"` + // ArtifactPackageTaskArgs arguments for artifact-package type tasks + ArtifactPackageTaskArgs *ArtifactPackageTaskArgs `bson:"artifact_package_args,omitempty" json:"artifact_package_args,omitempty"` + // ConfigPayload system config info + ConfigPayload *ConfigPayload `json:"config_payload,omitempty"` + Error string `bson:"error,omitempty" json:"error,omitempty"` + Services [][]*ProductService `bson:"services" json:"services"` + Render *RenderInfo `bson:"render" json:"render"` + StorageURI string `bson:"storage_uri,omitempty" json:"storage_uri,omitempty"` // interface{} 为types.TestReport - TestReports map[string]interface{} `bson:"test_reports,omitempty" json:"test_reports,omitempty"` - - RwLock sync.Mutex `bson:"-" json:"-"` - - ResetImage bool `json:"resetImage" bson:"resetImage"` - - TriggerBy *TriggerBy `json:"trigger_by,omitempty" bson:"trigger_by,omitempty"` - - Features []string `bson:"features" json:"features"` - IsRestart bool `bson:"is_restart" json:"is_restart"` - StorageEndpoint string `bson:"storage_endpoint" json:"storage_endpoint"` + TestReports map[string]interface{} `bson:"test_reports,omitempty" json:"test_reports,omitempty"` + RwLock sync.Mutex `bson:"-" json:"-"` + ResetImage bool `bson:"resetImage" json:"resetImage"` + ResetImagePolicy setting.ResetImagePolicyType `bson:"reset_image_policy" json:"reset_image_policy"` + TriggerBy *TriggerBy `json:"trigger_by,omitempty" bson:"trigger_by,omitempty"` + Features []string `bson:"features" json:"features"` + IsRestart bool `bson:"is_restart" json:"is_restart"` + StorageEndpoint string `bson:"storage_endpoint" json:"storage_endpoint"` + ArtifactInfo *ArtifactInfo `bson:"artifact_info" json:"artifact_info"` } type RenderInfo struct { @@ -122,6 +122,11 @@ type DeployArgs struct { PackageFile string `json:"package_file"` } +type CallbackArgs struct { + CallbackUrl string `bson:"callback_url" json:"callback_url"` // url-encoded full path + CallbackVars map[string]interface{} `bson:"callback_vars" json:"callback_vars"` // custom defied vars, will be set to body of callback request +} + type WorkflowTaskArgs struct { WorkflowName string `bson:"workflow_name" json:"workflow_name"` ProductTmplName string `bson:"product_tmpl_name" json:"product_tmpl_name"` @@ -137,6 +142,7 @@ type WorkflowTaskArgs struct { VersionArgs *VersionArgs `bson:"version_args,omitempty" json:"version_args,omitempty"` ReqID string `bson:"req_id" json:"req_id"` RegistryID string `bson:"registry_id,omitempty" json:"registry_id,omitempty"` + StorageID string `bson:"storage_id,omitempty" json:"storage_id,omitempty"` DistributeEnabled bool `bson:"distribute_enabled" json:"distribute_enabled"` WorklowTaskCreator string `bson:"workflow_task_creator" json:"workflow_task_creator"` // Ignore docker build cache @@ -153,13 +159,16 @@ type WorkflowTaskArgs struct { Source string `bson:"source" json:"source"` CodehostID int `bson:"codehost_id" json:"codehost_id"` RepoOwner string `bson:"repo_owner" json:"repo_owner"` + RepoNamespace string `bson:"repo_namespace" json:"repo_namespace"` RepoName string `bson:"repo_name" json:"repo_name"` - + Committer string `bson:"committer,omitempty" json:"committer,omitempty"` //github check run HookPayload *HookPayload `bson:"hook_payload" json:"hook_payload,omitempty"` // 请求模式，openAPI表示外部客户调用 RequestMode string `json:"request_mode,omitempty"` IsParallel bool `json:"is_parallel" bson:"is_parallel"` + + Callback *CallbackArgs `bson:"callback" json:"callback"` } type TestArgs struct { @@ -170,10 +179,15 @@ type TestArgs struct { } type ArtifactArgs struct { - Name string `bson:"name" json:"name"` - ServiceName string `bson:"service_name" json:"service_name"` - Image string `bson:"image" json:"image"` - Deploy []DeployEnv `bson:"deloy" json:"deploy"` + Name string `bson:"name" json:"name"` + ImageName string `bson:"image_name,omitempty" json:"image_name,omitempty"` + ServiceName string `bson:"service_name" json:"service_name"` + Image string `bson:"image" json:"image"` + Deploy []DeployEnv `bson:"deploy" json:"deploy"` + WorkflowName string `bson:"workflow_name,omitempty" json:"workflow_name,omitempty"` + TaskID int64 `bson:"task_id,omitempty" json:"task_id,omitempty"` + FileName string `bson:"file_name,omitempty" json:"file_name,omitempty"` + URL string `bson:"url,omitempty" json:"url,omitempty"` } type VersionArgs struct { @@ -200,16 +214,18 @@ type HookPayload struct { } type TargetArgs struct { - Name string `bson:"name" json:"name"` - ServiceName string `bson:"service_name" json:"service_name"` - Build *BuildArgs `bson:"build" json:"build"` - Version string `bson:"version" json:"version"` - Deploy []DeployEnv `bson:"deloy" json:"deploy"` - Image string `bson:"image" json:"image"` - BinFile string `bson:"bin_file" json:"bin_file"` - Envs []*KeyVal `bson:"envs" json:"envs"` - HasBuild bool `bson:"has_build" json:"has_build"` - JenkinsBuildArgs *JenkinsBuildArgs `bson:"jenkins_build_args" json:"jenkins_build_args"` + Name string `bson:"name" json:"name"` + ImageName string `bson:"image_name" json:"image_name"` + ServiceName string `bson:"service_name" json:"service_name"` + ServiceType string `bson:"service_type,omitempty" json:"service_type,omitempty"` + ProductName string `bson:"product_name" json:"product_name"` + Build *BuildArgs `bson:"build" json:"build"` + Deploy []DeployEnv `bson:"deploy" json:"deploy"` + Image string `bson:"image,omitempty" json:"image,omitempty"` + BinFile string `bson:"bin_file" json:"bin_file"` + Envs []*KeyVal `bson:"envs" json:"envs"` + HasBuild bool `bson:"has_build" json:"has_build"` + JenkinsBuildArgs *JenkinsBuildArgs `bson:"jenkins_build_args,omitempty" json:"jenkins_build_args,omitempty"` } type TestTaskArgs struct { @@ -243,6 +259,26 @@ type ServiceTaskArgs struct { Updatable bool `bson:"updatable" json:"updatable"` } +type ImageData struct { + ImageUrl string `bson:"image_url" json:"image_url" yaml:"image_url"` + ImageName string `bson:"image_name" json:"image_name" yaml:"image_name"` + ImageTag string `bson:"image_tag" json:"image_tag" yaml:"image_tag"` + CustomTag string `bson:"custom_tag" json:"custom_tag" yaml:"custom_tag"` + RegistryID string `bson:"registry_id" json:"registry_id" yaml:"registry_id"` +} + +type ImagesByService struct { + ServiceName string `bson:"service_name" json:"service_name" yaml:"service_name"` + Images []*ImageData `bson:"images" json:"images" yaml:"images"` +} + +type ArtifactPackageTaskArgs struct { + ProductName string `bson:"product_name" json:"product_name"` + Images []*ImagesByService `bson:"images" json:"images"` + SourceRegistries []string `bson:"source_registries" json:"source_registries"` + TargetRegistries []string `bson:"target_registries" json:"target_registries"` +} + type ProductService struct { ServiceName string `bson:"service_name" json:"service_name"` ProductName string `bson:"product_name" json:"product_name"` @@ -281,16 +317,6 @@ type TriggerBy struct { CommitID string `json:"commit_id,omitempty" bson:"commit_id,omitempty"` } -type Stage struct { - // 注意: 同一个stage暂时不能运行不同类型的Task - TaskType config.TaskType `bson:"type" json:"type"` - Status config.Status `bson:"status" json:"status"` - RunParallel bool `bson:"run_parallel" json:"run_parallel"` - Desc string `bson:"desc,omitempty" json:"desc,omitempty"` - SubTasks map[string]map[string]interface{} `bson:"sub_tasks" json:"sub_tasks"` - AfterAll bool `json:"after_all" bson:"after_all"` -} - func (Task) TableName() string { return "pipeline_task_v2" } diff --git a/pkg/microservice/warpdrive/core/service/types/task/release_image.go b/pkg/microservice/warpdrive/core/service/types/task/release_image.go index cb2a7e4d368fcdbe38ae8a6b8ac6056ccfeafb9a..2ae182d3d931148b86f048d664e6f97b79c3a967 100644 --- a/pkg/microservice/warpdrive/core/service/types/task/release_image.go +++ b/pkg/microservice/warpdrive/core/service/types/task/release_image.go @@ -43,6 +43,35 @@ type ReleaseImage struct { // destinations to distribute images Releases []RepoImage `bson:"releases" json:"releases"` + + // New Version field + ProductName string `bson:"product_name" json:"product_name"` + SourceImage string `bson:"source_image" json:"source_image"` + DistributeInfo []*DistributeInfo `bson:"distribute_info" json:"distribute_info"` +} + +// DistributeInfo will be convert into yaml, adding yaml +type DistributeInfo struct { + Image string `json:"image" yaml:"image"` + ReleaseName string `json:"release_name" yaml:"release_name"` + DistributeStartTime int64 `json:"distribute_start_time" yaml:"distribute_start_time"` + DistributeEndTime int64 `json:"distribute_end_time" yaml:"distribute_end_time"` + DistributeStatus string `json:"distribute_status" yaml:"distribute_status"` + DeployEnabled bool `json:"deploy_enabled" yaml:"deploy_enabled"` + DeployEnv string `json:"deploy_env" yaml:"deploy_env"` + DeployServiceType string `json:"deploy_service_type" yaml:"deploy_service_type"` + DeployContainerName string `json:"deploy_container_name" yaml:"deploy_container_name"` + DeployServiceName string `json:"deploy_service_name" yaml:"deploy_service_name"` + DeployClusterID string `json:"deploy_cluster_id" yaml:"deploy_cluster_id"` + DeployNamespace string `json:"deploy_namespace" yaml:"deploy_namespace"` + DeployStartTime int64 `json:"deploy_start_time" yaml:"deploy_start_time"` + DeployEndTime int64 `json:"deploy_end_time" yaml:"deploy_end_time"` + DeployStatus string `json:"deploy_status" yaml:"deploy_status"` + RepoID string `json:"repo_id" yaml:"repo_id"` + + //RepoAK and RepoSK is used for docker login, it is determined runtime, not passed by message so no json is required + RepoAK string `json:"-" yaml:"repo_ak"` + RepoSK string `json:"-" yaml:"repo_sk"` } type RepoImage struct { diff --git a/pkg/microservice/warpdrive/core/service/types/task/scanning.go b/pkg/microservice/warpdrive/core/service/types/task/scanning.go new file mode 100644 index 0000000000000000000000000000000000000000..02758cec016161c945cefde0435ae6b5f1d30626 --- /dev/null +++ b/pkg/microservice/warpdrive/core/service/types/task/scanning.go @@ -0,0 +1,46 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package task + +import ( + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/warpdrive/config" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/types" +) + +type Scanning struct { + TaskType config.TaskType `bson:"type" json:"type"` + Status config.Status `bson:"status" json:"status,omitempty"` + ScanningID string `bson:"scanning_id" json:"scanning_id"` + Name string `bson:"name" json:"name"` + Error string `bson:"error,omitempty" json:"error,omitempty"` + ImageInfo string `bson:"image_info" json:"image_info"` + SonarInfo *types.SonarInfo `bson:"sonar_info" json:"sonar_info"` + Repos []*types.Repository `bson:"repos" json:"repos"` + Proxy *models.Proxy `bson:"proxy" json:"proxy"` + ClusterID string `bson:"cluster_id" json:"cluster_id"` + // ResReq defines job requested resources + ResReq setting.Request `bson:"res_req" json:"res_req"` + ResReqSpec setting.RequestSpec `bson:"res_req_spec" json:"res_req_spec"` + Timeout int64 `bson:"timeout" json:"timeout"` + Registries []*RegistryNamespace `bson:"-" json:"registries"` + // Parameter is for sonarQube type only + Parameter string `bson:"parameter" json:"parameter"` + // Script is for other type only + Script string `bson:"script" json:"script"` +} diff --git a/pkg/microservice/warpdrive/core/service/types/task/testing.go b/pkg/microservice/warpdrive/core/service/types/task/testing.go index fcbb0a0c08256b9f25f38d0282bcc9e46f6c70ce..fca2eb8647f164cb88634322a547a1dfd8432861 100644 --- a/pkg/microservice/warpdrive/core/service/types/task/testing.go +++ b/pkg/microservice/warpdrive/core/service/types/task/testing.go @@ -21,6 +21,7 @@ import ( "github.com/koderover/zadig/pkg/microservice/warpdrive/config" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/types" ) type Testing struct { @@ -40,11 +41,20 @@ type Testing struct { ImageID string `bson:"image_id" json:"image_id"` // ResReq defines job requested resources ResReq setting.Request `bson:"res_req" json:"res_req"` + ResReqSpec setting.RequestSpec `bson:"res_req_spec" json:"res_req_spec"` LogFile string `bson:"log_file" json:"log_file"` TestModuleName string `bson:"test_module_name" json:"test_module_name"` ReportReady bool `bson:"report_ready" json:"report_ready"` IsRestart bool `bson:"is_restart" json:"is_restart"` Registries []*RegistryNamespace `bson:"-" json:"registries"` + ClusterID string `bson:"cluster_id" json:"cluster_id"` + Namespace string `bson:"namespace" json:"namespace"` + + // New since V1.10.0. + Cache types.Cache `bson:"cache" json:"cache"` + CacheEnable bool `bson:"cache_enable" json:"cache_enable"` + CacheDirType types.CacheDirType `bson:"cache_dir_type" json:"cache_dir_type"` + CacheUserDir string `bson:"cache_user_dir" json:"cache_user_dir"` } func (t *Testing) ToSubTask() (map[string]interface{}, error) { diff --git a/pkg/microservice/warpdrive/core/service/types/task/trigger.go b/pkg/microservice/warpdrive/core/service/types/task/trigger.go new file mode 100644 index 0000000000000000000000000000000000000000..427b7226cb126e78fee1f53e3817b409860b5eee --- /dev/null +++ b/pkg/microservice/warpdrive/core/service/types/task/trigger.go @@ -0,0 +1,81 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package task + +import ( + "fmt" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/warpdrive/config" +) + +type Trigger struct { + TaskType config.TaskType `bson:"type" json:"type"` + Enabled bool `bson:"enabled" json:"enabled"` + TaskStatus config.Status `bson:"status" json:"status"` + URL string `bson:"url,omitempty" json:"url,omitempty"` + Path string `bson:"path,omitempty" json:"path,omitempty"` + IsCallback bool `bson:"is_callback,omitempty" json:"is_callback,omitempty"` + CallbackType string `bson:"callback_type,omitempty" json:"callback_type,omitempty"` + CallbackPayload *CallbackPayload `bson:"callback_payload,omitempty" json:"callback_payload,omitempty"` + Headers []*models.KeyVal `bson:"headers,omitempty" json:"headers,omitempty"` + Timeout int `bson:"timeout" json:"timeout,omitempty"` + IsRestart bool `bson:"is_restart" json:"is_restart"` + Error string `bson:"error,omitempty" json:"error,omitempty"` + StartTime int64 `bson:"start_time" json:"start_time,omitempty"` + EndTime int64 `bson:"end_time" json:"end_time,omitempty"` +} + +type CallbackPayload struct { + QRCodeURL string `bson:"QR_code_URL,omitempty" json:"QR_code_URL,omitempty"` +} + +// ToSubTask ... +func (t *Trigger) ToSubTask() (map[string]interface{}, error) { + var task map[string]interface{} + if err := IToi(t, &task); err != nil { + return nil, fmt.Errorf("convert trigger to interface error: %s", err) + } + return task, nil +} + +type WebhookPayload struct { + EventName string `json:"event_name"` + ProjectName string `json:"project_name"` + TaskName string `json:"task_name"` + TaskID int64 `json:"task_id"` + TaskOutput []*TaskOutput `json:"task_output,omitempty"` + TaskEnvs []*KeyVal `json:"task_envs,omitempty"` + ServiceInfos []*ServiceInfo `json:"service_infos"` + Creator string `json:"creator"` + WorkflowStatus config.Status `json:"workflow_status"` +} + +type TaskOutput struct { + Type string `json:"type"` + Value string `json:"value"` +} + +type CallbackPayloadObj struct { + TaskName string `json:"task_name"` + ProjectName string `json:"project_name"` + TaskID int `json:"task_id"` + Type string `json:"type"` + Status string `json:"status"` + StatusMessage string `json:"status_message"` + Payload *CallbackPayload `json:"payload"` +} diff --git a/pkg/microservice/warpdrive/core/service/types/task/warpdrive.go b/pkg/microservice/warpdrive/core/service/types/task/warpdrive.go index 15a7914577d6c69fa4978f39e157adc33b8d3f10..20c84420761ff502e87a4e5b8a71d5bbd4f3032d 100644 --- a/pkg/microservice/warpdrive/core/service/types/task/warpdrive.go +++ b/pkg/microservice/warpdrive/core/service/types/task/warpdrive.go @@ -16,6 +16,10 @@ limitations under the License. package task +import ( + "github.com/koderover/zadig/pkg/types" +) + type PipelineCtx struct { DockerHost string Workspace string @@ -23,6 +27,12 @@ type PipelineCtx struct { DockerMountDir string ConfigMapMountDir string MultiRun bool + + // New since V1.10.0. + Cache types.Cache + CacheEnable bool + CacheDirType types.CacheDirType + CacheUserDir string } type JobCtxBuilder struct { diff --git a/pkg/microservice/warpdrive/server/server.go b/pkg/microservice/warpdrive/server/server.go index 329480f338c51848a7a039d20294025b9b9b86a4..f2c1241af1f4e711ab40f181f71422a81837255f 100644 --- a/pkg/microservice/warpdrive/server/server.go +++ b/pkg/microservice/warpdrive/server/server.go @@ -18,6 +18,7 @@ package server import ( "context" + "fmt" "net/http" "time" @@ -37,8 +38,10 @@ func Serve(ctx context.Context) error { log.Info("Warpdrive service start ... ") - if err := taskcontroller.InitTaskController(ctx); err != nil { - log.Fatalf("NewTaskController error: %v", err) + controller := taskcontroller.NewController() + err := controller.Init(ctx) + if err != nil { + return fmt.Errorf("failed to init controller: %s", err) } http.HandleFunc("/ping", ping) @@ -65,7 +68,7 @@ func Serve(ctx context.Context) error { <-stopChan - return nil + return controller.Stop(ctx) } func ping(w http.ResponseWriter, r *http.Request) { diff --git a/pkg/microservice/zgctl/handler.go b/pkg/microservice/zgctl/handler.go new file mode 100644 index 0000000000000000000000000000000000000000..4940792cb421b85f9e7a1ed2986545ba0c6adc29 --- /dev/null +++ b/pkg/microservice/zgctl/handler.go @@ -0,0 +1,111 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package zgctl + +import ( + "errors" + + "github.com/dgraph-io/badger/v3" + "github.com/gin-gonic/gin" + + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + "github.com/koderover/zadig/pkg/types" +) + +type HandlerConfig struct { + ZadigHost string + ZadigToken string + HomeDir string + DB *badger.DB +} + +type handlers struct { + zgctl ZgCtler +} + +type KubeconfigInfo struct { + KubeConfig KubeConfigData `json:"kubeconfig"` +} + +type KubeConfigData struct { + Path string `json:"path"` +} + +func NewHandlers(cfg *HandlerConfig) ZgCtlHandler { + zgctl := NewZgCtl(&ZgCtlConfig{ + ZadigHost: cfg.ZadigHost, + ZadigToken: cfg.ZadigToken, + HomeDir: cfg.HomeDir, + DB: cfg.DB, + }) + + return &handlers{ + zgctl: zgctl, + } +} + +func (h *handlers) StartDevMode(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + envName := c.Param("envName") + serviceName := c.Param("serviceName") + projectName := c.Query("projectName") + dataDir := c.Query("dataDir") + + var startInfo types.StartDevmodeInfo + if err := c.BindJSON(&startInfo); err != nil { + ctx.Err = errors.New("invalid request body") + return + } + + ctx.Err = h.zgctl.StartDevMode(c, projectName, envName, serviceName, dataDir, startInfo.DevImage) +} + +func (h *handlers) StopDevMode(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + envName := c.Param("envName") + serviceName := c.Param("serviceName") + projectName := c.Query("projectName") + + ctx.Err = h.zgctl.StopDevMode(c, projectName, envName, serviceName) +} + +func (h *handlers) DevImages(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp = h.zgctl.DevImages() +} + +func (h *handlers) ConfigKubeconfig(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + envName := c.Param("envName") + projectName := c.Query("projectName") + + var kubeconfig KubeconfigInfo + if err := c.BindJSON(&kubeconfig); err != nil { + ctx.Err = errors.New("invalid request body") + return + } + + ctx.Err = h.zgctl.ConfigKubeconfig(projectName, envName, kubeconfig.KubeConfig.Path) +} diff --git a/pkg/microservice/zgctl/interface.go b/pkg/microservice/zgctl/interface.go new file mode 100644 index 0000000000000000000000000000000000000000..8444e3ea3d1375ca12f78ea70f446f3a8db63e03 --- /dev/null +++ b/pkg/microservice/zgctl/interface.go @@ -0,0 +1,51 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package zgctl + +import ( + "context" + + "github.com/gin-gonic/gin" +) + +type ZgCtler interface { + // StartDevMode starts dev mode. + StartDevMode(ctx context.Context, projectName, envName, serviceName, dataDir, devImage string) error + + // StopDevMode stops dev mode. + StopDevMode(ctx context.Context, projectName, envName, serviceName string) error + + // DevImages provides image list for dev mode. + DevImages() []string + + // ConfigKubeconfig stores relations between env and kubeconfig. + ConfigKubeconfig(projectName, envName, kubeconfigPath string) error +} + +type ZgCtlHandler interface { + // StartDevMode is a gin handler that starts dev mode. + StartDevMode(*gin.Context) + + // StopDevMode is a gin handler that stops dev mode. + StopDevMode(*gin.Context) + + // DevImages is a gin handler that provides image list for dev mode. + DevImages(*gin.Context) + + // ConfigKubeconfig is a gin handler that stores relations between env and kubeconfig. + ConfigKubeconfig(*gin.Context) +} diff --git a/pkg/microservice/zgctl/router.go b/pkg/microservice/zgctl/router.go new file mode 100644 index 0000000000000000000000000000000000000000..0f3ebfc5edc96745977e664944bc603ad3456465 --- /dev/null +++ b/pkg/microservice/zgctl/router.go @@ -0,0 +1,54 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package zgctl + +import ( + "github.com/dgraph-io/badger/v3" + "github.com/gin-gonic/gin" +) + +type RouterConfig struct { + ZadigHost string + ZadigToken string + HomeDir string + DB *badger.DB +} + +type Router struct { + handlers ZgCtlHandler +} + +func NewRouter(cfg *RouterConfig) *Router { + handlers := NewHandlers(&HandlerConfig{ + ZadigHost: cfg.ZadigHost, + ZadigToken: cfg.ZadigToken, + HomeDir: cfg.HomeDir, + DB: cfg.DB, + }) + + return &Router{ + handlers: handlers, + } +} + +func (r *Router) Inject(router *gin.RouterGroup) { + router.POST("/devmode/:envName/:serviceName", r.handlers.StartDevMode) + router.DELETE("/devmode/:envName/:serviceName", r.handlers.StopDevMode) + router.POST("/devmode/:envName/kubeconfig", r.handlers.ConfigKubeconfig) + + router.GET("/devmode/images", r.handlers.DevImages) +} diff --git a/pkg/microservice/zgctl/template.go b/pkg/microservice/zgctl/template.go new file mode 100644 index 0000000000000000000000000000000000000000..ad56ca0962a75cfe62466270ac3cf25b422c8380 --- /dev/null +++ b/pkg/microservice/zgctl/template.go @@ -0,0 +1,186 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package zgctl + +var syncthingConfigTmpl = ` + + + basic + + + + + + + 1 + + 3600 + + basic + + 0 + 0 + 0 + random + false + 0 + 0 + 10 + false + false + false + 25 + .stfolder + false + 0 + 2 + false + standard + standard + false + false + + +
dynamic
+ false + false + 0 + 0 + 0 + false + 0 + + +
{{.RemoteAddr}}
+ false + false + 0 + 0 + 0 + + +
127.0.0.1:8384
+ Tb4ahP7HEXxqDwf6UehnySgcb9mUozuM + default + + + + {{.ListenAddr}} + default + false + false + 21027 + [ff12::8384]:21027 + 0 + 0 + 60 + false + 10 + false + true + 60 + 30 + 10 + -1 + 3 + + https://data.syncthing.net/newdata + false + 1800 + true + 12 + false + 24 + false + 5 + false + 1 + https://upgrades.syncthing.net/meta.json + false + 10 + authenticationUserAndPassword + 0 + true + 0 + https://crash.syncthing.net/newcrash + true + 180 + 20 + default + auto + 0 + true + false + 0 + 0 + false + + + + basic + + + + 1 + + 3600 + + basic + + 0 + 0 + 0 + random + false + 0 + 0 + 10 + false + false + false + 25 + .stfolder + false + 0 + 2 + false + standard + standard + false + false + + +
dynamic
+ false + false + 0 + 0 + 0 + false + 0 + + + + + +` + +var syncthingRunTmpl = ` +#!/bin/sh + +nohup {{.SyncthingBin}} serve --config={{.ConfigDir}} --data={{.DataDir}} & +` diff --git a/pkg/microservice/zgctl/utils.go b/pkg/microservice/zgctl/utils.go new file mode 100644 index 0000000000000000000000000000000000000000..2319ebd4e6a60fc4db67b41e5e82c7120ca2293d --- /dev/null +++ b/pkg/microservice/zgctl/utils.go @@ -0,0 +1,27 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package zgctl + +var devImages = []string{ + "koderover.tencentcloudcr.com/koderover-public/golang:1.18.1", + "koderover.tencentcloudcr.com/koderover-public/node:18.2", + "koderover.tencentcloudcr.com/koderover-public/python:3.10.4-buster", + "koderover.tencentcloudcr.com/koderover-public/openjdk:19-jdk-buster", + "koderover.tencentcloudcr.com/koderover-public/php:8.1.6-fpm", + "koderover.tencentcloudcr.com/koderover-public/ruby:3.1.2", + "koderover.tencentcloudcr.com/koderover-public/rust:1.61.0", +} diff --git a/pkg/microservice/zgctl/zgctl.go b/pkg/microservice/zgctl/zgctl.go new file mode 100644 index 0000000000000000000000000000000000000000..77a25529ad3b7f77d801ffeb8baba710bde98b88 --- /dev/null +++ b/pkg/microservice/zgctl/zgctl.go @@ -0,0 +1,473 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package zgctl + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io/ioutil" + "math/rand" + "os" + "os/exec" + "path/filepath" + "strings" + "text/template" + "time" + + "github.com/dgraph-io/badger/v3" + "github.com/shirou/gopsutil/v3/process" + + "github.com/koderover/zadig/pkg/shared/client/aslan" + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types" +) + +type ZgCtlConfig struct { + ZadigHost string + ZadigToken string + HomeDir string + DB *badger.DB +} + +type zgctl struct { + aslanClient *aslan.Client + homeDir string + syncthingBin string + db *badger.DB +} + +type SyncthingConfig struct { + SyncDirPath string + SyncType string + LocalDeviceID string + RemoteDeviceID string + RemoteAddr string + ListenAddr string +} + +type SyncthingScript struct { + SyncthingBin string + ConfigDir string + DataDir string +} + +func init() { + rand.Seed(time.Now().UTC().UnixNano()) +} + +func NewZgCtl(cfg *ZgCtlConfig) ZgCtler { + aslanClient := aslan.NewExternal(cfg.ZadigHost, cfg.ZadigToken) + aslanClient.SetTimeout(3 * time.Minute) + + syncthingBin := filepath.Join(cfg.HomeDir, "bin", "syncthing") + log.Infof("home dir: %s, syncthing bin: %s", cfg.HomeDir, syncthingBin) + + return &zgctl{ + aslanClient: aslanClient, + homeDir: cfg.HomeDir, + syncthingBin: syncthingBin, + db: cfg.DB, + } +} + +func (z *zgctl) StartDevMode(ctx context.Context, projectName, envName, serviceName, dataDir, devImage string) error { + // 1. Adjust replicas of the workload to 1 and patch Pod. + log.Info("Begin to patch worload.") + workloadInfo, err := z.aslanClient.PatchWorkload(projectName, envName, serviceName, devImage) + if err != nil { + errRet := fmt.Errorf("failed to patch service %q in env %q of project %q: %s", envName, serviceName, projectName, err) + log.Error(errRet) + return errRet + } + + log.Infof("workload info: %#v", workloadInfo) + err = z.setZadig2K8sData(projectName, envName, serviceName, workloadInfo) + if err != nil { + errRet := fmt.Errorf("failed to set workload info: %s", err) + log.Error(errRet) + return errRet + } + + // 2. Configure `Syncthing`. + log.Info("Begin to config Syncthing.") + localPort := z.genRandPort() + remotePort := z.genRandPort() + configDir := z.generateServiceConfigDir(projectName, envName, serviceName) + err = z.configSyncthing(ctx, configDir, dataDir, localPort, remotePort) + if err != nil { + errRet := fmt.Errorf("failed to config syncthing for service %q in env %q: %s", envName, serviceName, err) + log.Error(errRet) + return errRet + } + + // 3. Execute port-forward to implement one-way network communication. + log.Info("Begin to do port-forward.") + + kubeconfigPath, err := z.getEnvKubeconfigData(projectName, envName) + if err != nil { + errRet := fmt.Errorf("failed to find kubeconfig path of env %q in project %q: %s", envName, projectName, err) + log.Error(errRet) + return errRet + } + log.Infof("kubeconfig path for env %q in project %q: %s", envName, projectName, kubeconfigPath) + + err = z.podPortForward(ctx, workloadInfo.PodNamespace, workloadInfo.PodName, kubeconfigPath, remotePort) + if err != nil { + errRet := fmt.Errorf("failed to port-forward service %q in env %q: %s", envName, serviceName, err) + log.Error(errRet) + return errRet + } + + // 4. Start `Syncthing`. + log.Info("Begin to start Syncthing.") + err = z.startSyncthing(ctx, workloadInfo.PodNamespace, workloadInfo.PodName, configDir, dataDir, kubeconfigPath) + if err != nil { + errRet := fmt.Errorf("failed to start syncthing: %s", err) + log.Error(errRet) + return errRet + } + + return nil +} + +func (z *zgctl) StopDevMode(ctx context.Context, projectName, envName, serviceName string) error { + configDir := z.generateServiceConfigDir(projectName, envName, serviceName) + + // 1. Stop `Syncthing` locally and delete the corresponding configuration. + log.Info("Begin to stop Syncthing.") + + workloadInfo, err := z.getZadig2K8sData(projectName, envName, serviceName) + if err != nil { + errRet := fmt.Errorf("failed to get find workload info for project %q, envName %q, serviceName %q: %s", projectName, envName, serviceName, err) + log.Error(errRet) + return errRet + } + + err = z.stopSyncthing(ctx, configDir, workloadInfo) + if err != nil { + log.Warnf("Failed to stop syncthing for service %q in env %q: %s", envName, serviceName, err) + } + + // 2. Recover workload. + log.Info("Begin to recover workload.") + err = z.aslanClient.RecoverWorkload(projectName, envName, serviceName) + if err != nil { + errRet := fmt.Errorf("failed to recover workload: %s", err) + log.Error(errRet) + return errRet + } + + return nil +} + +func (z *zgctl) DevImages() []string { + return devImages +} + +func (z *zgctl) ConfigKubeconfig(projectName, envName, kubeconfigPath string) error { + return z.setEnvKubeconfigData(projectName, envName, kubeconfigPath) +} + +func (z *zgctl) configSyncthing(ctx context.Context, configDir, dataDir string, localPort, remotePort int) error { + // 1. Generate local `Syncthing` config. + localDeviceID, err := z.configLocalSyncthing(configDir) + if err != nil { + return fmt.Errorf("failed to generate local Syncthing: %s", err) + } + log.Infof("local device id: %s", localDeviceID) + + // 2. Generate remote `Syncthing` config. + tmpRemoteConfigPath := filepath.Join(configDir, "remote") + tmpRemoteDeviceID, err := z.configLocalSyncthing(tmpRemoteConfigPath) + if err != nil { + return fmt.Errorf("failed to generate tmp remote Syncthing: %s", err) + } + log.Infof("tmp remote device id: %s", tmpRemoteDeviceID) + + // 3. Adjust configs of local and remote Syncthing. + return z.adjustSyncthingConfig(localDeviceID, tmpRemoteDeviceID, configDir, dataDir, localPort, remotePort) +} + +func (z *zgctl) startSyncthing(ctx context.Context, ns, podName, configDir, dataDir, kubeconfigPath string) error { + // 1. Upload config to remote Pod and start Syncthing in remote Pod. + err := z.startRemoteSyncthing(ctx, ns, podName, configDir, kubeconfigPath) + if err != nil { + return fmt.Errorf("failed to start remote syncthing: %s", err) + } + + // 2. Start local Syncthing. + return z.startLocalSyncthing(ctx, configDir, dataDir) +} + +func (z *zgctl) stopSyncthing(ctx context.Context, configDir string, workloadInfo *types.WorkloadInfo) error { + processes, err := process.Processes() + if err != nil { + return err + } + + for _, p := range processes { + // Note: Don't use `p.Name()`. See: https://github.com/shirou/gopsutil/issues/1043 + cmdline, err := p.Cmdline() + if err != nil { + log.Warnf("Failed to find command line: %s", err) + continue + } + + if (strings.Contains(cmdline, "syncthing serve") && strings.Contains(cmdline, configDir)) || + strings.Contains(cmdline, fmt.Sprintf("kubectl -n %s port-forward %s", workloadInfo.PodNamespace, workloadInfo.PodName)) { + log.Infof("Begin to kill process %s. Cmdline: %s.", p, cmdline) + err = p.Kill() + if err != nil { + log.Warnf("Failed to kill process %s: %s", p, err) + } + } + } + + return os.RemoveAll(configDir) +} + +func (z *zgctl) podPortForward(ctx context.Context, ns, podName, kubeconfigPath string, port int) error { + go func() { + portForward := fmt.Sprintf("kubectl -n %s port-forward %s %d:%d --kubeconfig %s", ns, podName, port, port, kubeconfigPath) + log.Infof("port-forward: %s", portForward) + + cmd := exec.Command("/bin/sh", "-c", portForward) + res, err := cmd.CombinedOutput() + if err != nil { + log.Errorf("Failed to do port-forward %q: %s. Msg: %s.", portForward, err, string(res)) + return + } + + log.Infof("Has done `%q`: %s.", portForward, string(res)) + }() + + return nil +} + +func (z *zgctl) configLocalSyncthing(configDir string) (deviceID string, err error) { + log.Infof("config dir: %s", configDir) + + var errBuffer bytes.Buffer + cmd := exec.Command("/bin/sh", "-c", fmt.Sprintf("%s generate --config=%s", z.syncthingBin, configDir)) + cmd.Stderr = &errBuffer + err = cmd.Run() + if err != nil { + return "", fmt.Errorf("%s %s", err, errBuffer.String()) + } + + items := strings.Split(errBuffer.String(), "Device ID:") + return strings.TrimSpace(items[1]), nil +} + +func (z *zgctl) generateServiceConfigDir(projectName, envName, serviceName string) string { + return filepath.Join(z.homeDir, "config", fmt.Sprintf("%s-%s-%s", projectName, envName, serviceName)) +} + +func (z *zgctl) adjustSyncthingConfig(localDeviceID, remoteDeviceID, configDir, dataDir string, localPort, remotePort int) error { + configTmpl, err := template.New("local").Parse(syncthingConfigTmpl) + if err != nil { + return fmt.Errorf("failed to parse config template: %s", err) + } + + localAddr := fmt.Sprintf("tcp://127.0.0.1:%d", localPort) + remoteAddr := fmt.Sprintf("tcp://127.0.0.1:%d", remotePort) + + localData := SyncthingConfig{ + SyncDirPath: dataDir, + SyncType: "sendonly", + LocalDeviceID: localDeviceID, + RemoteDeviceID: remoteDeviceID, + RemoteAddr: remoteAddr, + ListenAddr: localAddr, + } + + var localBuffer bytes.Buffer + err = configTmpl.Execute(&localBuffer, localData) + if err != nil { + return fmt.Errorf("failed to execute local config template: %s", err) + } + + err = ioutil.WriteFile(filepath.Join(configDir, "config.xml"), localBuffer.Bytes(), 0600) + if err != nil { + return fmt.Errorf("failed to overwrite local config: %s", err) + } + + scriptTmpl, err := template.New("local-script").Parse(syncthingRunTmpl) + if err != nil { + return fmt.Errorf("failed to parse script template: %s", err) + } + + localScriptData := SyncthingScript{ + SyncthingBin: filepath.Join(z.homeDir, "bin", "syncthing"), + ConfigDir: configDir, + DataDir: dataDir, + } + + var localScriptBuffer bytes.Buffer + err = scriptTmpl.Execute(&localScriptBuffer, localScriptData) + if err != nil { + return fmt.Errorf("failed to execute local script template: %s", err) + } + + err = ioutil.WriteFile(filepath.Join(configDir, "run.sh"), localScriptBuffer.Bytes(), 0755) + if err != nil { + return fmt.Errorf("failed to write local run.sh: %s", err) + } + + remoteData := SyncthingConfig{ + SyncDirPath: types.DevmodeWorkDir, + SyncType: "receiveonly", + LocalDeviceID: localDeviceID, + RemoteDeviceID: remoteDeviceID, + RemoteAddr: "dynamic", + ListenAddr: remoteAddr, + } + + var remoteBuffer bytes.Buffer + err = configTmpl.Execute(&remoteBuffer, remoteData) + if err != nil { + return fmt.Errorf("failed to execute remote config template: %s", err) + } + + err = ioutil.WriteFile(filepath.Join(configDir, "remote", "config.xml"), remoteBuffer.Bytes(), 0600) + if err != nil { + return fmt.Errorf("failed to overwrite remote config: %s", err) + } + + remoteScriptData := SyncthingScript{ + SyncthingBin: "syncthing", + ConfigDir: filepath.Join(types.DevmodeWorkDir, "config"), + DataDir: types.DevmodeWorkDir, + } + + var remoteScriptBuffer bytes.Buffer + err = scriptTmpl.Execute(&remoteScriptBuffer, remoteScriptData) + if err != nil { + return fmt.Errorf("failed to execute remote script template: %s", err) + } + + return ioutil.WriteFile(filepath.Join(configDir, "remote", "run.sh"), remoteScriptBuffer.Bytes(), 0755) +} + +func (z *zgctl) genRandPort() int { + max := 60000 + min := 40000 + + return rand.Intn(max-min) + min +} + +func (z *zgctl) genSyncthingServeAddr() string { + return fmt.Sprintf("tcp://127.0.0.1:%d", z.genRandPort()) +} + +func (z *zgctl) startRemoteSyncthing(ctx context.Context, ns, podName, configDir, kubeconfigPath string) error { + cpCmd := fmt.Sprintf("kubectl --kubeconfig %s -n %s cp %s %s:%s -c %s", kubeconfigPath, ns, filepath.Join(configDir, "remote"), podName, filepath.Join(types.DevmodeWorkDir, "config"), types.IDEContainerNameSidecar) + log.Infof("cp cmd: %s", cpCmd) + + cmd := exec.Command("/bin/sh", "-c", cpCmd) + res, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to cp files to remote pod: %s. msg: %s.", err, string(res)) + } + + execCmd := fmt.Sprintf("kubectl --kubeconfig %s -n %s exec %s -c %s -- /bin/sh -c '%s >/dev/null 2>/dev/null &'", kubeconfigPath, ns, podName, types.IDEContainerNameSidecar, filepath.Join(types.DevmodeWorkDir, "config", "run.sh")) + log.Infof("exec cmd: %s", execCmd) + cmd = exec.Command("/bin/sh", "-c", execCmd) + res, err = cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to run syncthing in remote pod: %s. msg: %s", err, string(res)) + } + + return nil +} + +func (z *zgctl) startLocalSyncthing(ctx context.Context, configDir, dataDir string) error { + script := fmt.Sprintf("%s >/dev/null 2>/dev/null &", filepath.Join(configDir, "run.sh")) + log.Infof("script: %s", script) + + cmd := exec.Command("/bin/sh", "-c", script) + res, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to run syncthing locally: %s. msg: %s.", err, string(res)) + } + + return nil +} + +func (z *zgctl) genZadig2K8sMapKey(projectName, envName, serviceName string) string { + return fmt.Sprintf("%s-%s-%s", projectName, envName, serviceName) +} + +func (z *zgctl) genEnvKubeconfigKey(projectName, envName string) string { + return fmt.Sprintf("%s-%s", projectName, envName) +} + +func (z *zgctl) setZadig2K8sData(projectName, envName, serviceName string, data *types.WorkloadInfo) error { + key := z.genZadig2K8sMapKey(projectName, envName, serviceName) + val, err := json.Marshal(data) + if err != nil { + return err + } + + return z.setData([]byte(key), val) +} + +func (z *zgctl) getZadig2K8sData(projectName, envName, serviceName string) (*types.WorkloadInfo, error) { + key := z.genZadig2K8sMapKey(projectName, envName, serviceName) + val, err := z.getData([]byte(key)) + if err != nil { + return nil, err + } + + var data types.WorkloadInfo + err = json.Unmarshal(val, &data) + return &data, err +} + +func (z *zgctl) setEnvKubeconfigData(projectName, envName, kubeconfigPath string) error { + key := z.genEnvKubeconfigKey(projectName, envName) + return z.setData([]byte(key), []byte(kubeconfigPath)) +} + +func (z *zgctl) getEnvKubeconfigData(projectName, envName string) (string, error) { + key := z.genEnvKubeconfigKey(projectName, envName) + val, err := z.getData([]byte(key)) + return string(val), err +} + +func (z *zgctl) setData(key, val []byte) error { + return z.db.Update(func(txn *badger.Txn) error { + return txn.Set(key, val) + }) +} + +func (z *zgctl) getData(key []byte) ([]byte, error) { + var val []byte + err := z.db.View(func(txn *badger.Txn) error { + item, err := txn.Get(key) + if err != nil { + return err + } + + val, err = item.ValueCopy(nil) + return err + }) + + return val, err +} diff --git a/pkg/middleware/gin/audit_log.go b/pkg/middleware/gin/audit_log.go index d4c79a4a0b9cd7c76258d02f0d1483ec7336e201..df1040ea84e5ecec5cac013bb58a175c1c8ee726 100644 --- a/pkg/middleware/gin/audit_log.go +++ b/pkg/middleware/gin/audit_log.go @@ -19,19 +19,26 @@ package gin import ( "github.com/gin-gonic/gin" - "github.com/koderover/zadig/pkg/config" - "github.com/koderover/zadig/pkg/shared/client/aslan" + systemservice "github.com/koderover/zadig/pkg/microservice/aslan/core/system/service" "github.com/koderover/zadig/pkg/util/ginzap" ) +// OperationLogStatus update status of operation if necessary +func OperationLogStatus() gin.HandlerFunc { + return func(c *gin.Context) { + defer UpdateOperationLogStatus(c) + c.Next() + } +} + // 更新操作日志状态 func UpdateOperationLogStatus(c *gin.Context) { - log := ginzap.WithContext(c).Sugar() c.Next() if c.GetString("operationLogID") == "" { return } - err := aslan.New(config.AslanServiceAddress(), config.PoetryAPIRootKey()).UpdateAuditLog(c.GetString("operationLogID"), c.Writer.Status(), log) + log := ginzap.WithContext(c).Sugar() + err := systemservice.UpdateOperation(c.GetString("operationLogID"), c.Writer.Status(), log) if err != nil { log.Errorf("UpdateOperation err:%v", err) } diff --git a/pkg/middleware/gin/auth.go b/pkg/middleware/gin/auth.go deleted file mode 100644 index 515781639f4838789e8e5d1b73075ecc29fdf3ce..0000000000000000000000000000000000000000 --- a/pkg/middleware/gin/auth.go +++ /dev/null @@ -1,84 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package gin - -import ( - "net/http" - "strings" - - "github.com/gin-gonic/gin" - - "github.com/koderover/zadig/pkg/config" - "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/poetry" - "github.com/koderover/zadig/pkg/tool/log" -) - -// Auth 返回会判断 Session 的中间件 -func Auth() gin.HandlerFunc { - return func(c *gin.Context) { - authorization := c.Request.Header.Get(setting.AuthorizationHeader) - if authorization != "" { - if strings.Contains(authorization, setting.RootAPIKey) { - token := strings.Split(authorization, " ") - if len(token) == 2 && token[1] == config.PoetryAPIRootKey() { - c.Set(setting.SessionUsername, "fake-user") - c.Set(setting.SessionUser, &poetry.UserInfo{Name: "fake-user", IsSuperUser: true, IsAdmin: true, OrganizationID: 1}) - c.Next() - return - } - } - if strings.Contains(authorization, setting.TIMERAPIKEY) { - token := strings.Split(authorization, " ") - if len(token) == 2 && token[1] == config.PoetryAPIRootKey() { - c.Set(setting.SessionUsername, "timer") - c.Set(setting.SessionUser, &poetry.UserInfo{Name: "timer", IsSuperUser: true, IsAdmin: true, OrganizationID: 1}) - c.Next() - return - } - } - if strings.Contains(authorization, setting.UserAPIKey) { - token := strings.Split(authorization, " ") - if len(token) == 2 { - userInfo, err := poetry.GetUserDetailByToken(config.PoetryServiceAddress(), token[1]) - if err != nil { - log.Errorf("get user detail err :%v", err) - c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"message": "get user detail error"}) - return - } - c.Set(setting.SessionUsername, userInfo.Name) - c.Set(setting.SessionUser, userInfo) - c.Next() - return - } - } - } else if token, err := c.Cookie("TOKEN"); err == nil { - userInfo, err := poetry.GetUserDetailByToken(config.PoetryServiceAddress(), token) - if err != nil { - log.Errorf("get user detail err :%v", err) - c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"message": "get user detail error"}) - return - } - c.Set(setting.SessionUsername, userInfo.Name) - c.Set(setting.SessionUser, userInfo) - c.Next() - return - } - - c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"message": "auth failed"}) - } -} diff --git a/pkg/middleware/gin/collaboration.go b/pkg/middleware/gin/collaboration.go new file mode 100644 index 0000000000000000000000000000000000000000..19800f0ad6070c4cee239366321ba33fbbf6d890 --- /dev/null +++ b/pkg/middleware/gin/collaboration.go @@ -0,0 +1,70 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gin + +import ( + "strings" + + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/collaboration/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" +) + +func GetCollaborationNew() gin.HandlerFunc { + return func(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + projectName := c.Query("projectName") + ifPassFilter := c.Query("ifPassFilter") + if ifPassFilter == "true" || + projectName == "" || + c.Request.URL.Path == "/api/collaboration/collaborations/sync" || + strings.Contains(c.Request.URL.Path, "/estimated-values") || + c.Request.URL.Path == "/api/aslan/environment/rendersets/default-values" { + c.Next() + return + } + newResp, err := service.GetCollaborationNew(projectName, ctx.UserID, ctx.IdentityType, ctx.Account, ctx.Logger) + if err != nil { + ctx.Err = err + c.Abort() + return + } + if newResp == nil { + c.Next() + return + } + if newResp.IfSync { + err := service.SyncCollaborationInstance(nil, projectName, ctx.UserID, ctx.IdentityType, ctx.Account, ctx.RequestID, ctx.Logger) + if err != nil { + ctx.Err = err + c.Abort() + return + } + } + if len(newResp.Product) == 0 && len(newResp.Workflow) == 0 { + c.Next() + return + } else { + ctx.Resp = newResp + c.Abort() + return + } + } +} diff --git a/pkg/middleware/gin/permission.go b/pkg/middleware/gin/permission.go deleted file mode 100644 index 8f5a690c43638de62c33ac400b5a0cc37c0486fc..0000000000000000000000000000000000000000 --- a/pkg/middleware/gin/permission.go +++ /dev/null @@ -1,190 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package gin - -import ( - "fmt" - "net/http" - - "github.com/gin-gonic/gin" - "k8s.io/apimachinery/pkg/util/sets" - - "github.com/koderover/zadig/pkg/config" - "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/poetry" - "github.com/koderover/zadig/pkg/types/permission" - "github.com/koderover/zadig/pkg/util/ginzap" -) - -// RequireSuperAdminAuth require super user role -func RequireSuperAdminAuth(c *gin.Context) { - log := ginzap.WithContext(c).Sugar() - username := c.GetString(setting.SessionUsername) - user, err := authUser(username, c) - if err != nil { - log.Errorf("authUser(%s) failed, %v", username, err) - c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"message": "Authentication failed."}) - return - } - if user.IsSuperUser { - c.Next() - return - } - c.AbortWithStatusJSON(http.StatusForbidden, gin.H{"message": "Require super user permission"}) -} - -//判断用户是否有某个权限 -func IsHavePermission(permissionUUIDs []string, paramType int) gin.HandlerFunc { - return func(c *gin.Context) { - log := ginzap.WithContext(c).Sugar() - username := c.GetString(setting.SessionUsername) - user, err := authUser(username, c) - if err != nil { - log.Errorf("authUser(%s) failed, %v", username, err) - c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"message": "Authentication failed."}) - return - } - - if user.IsSuperUser { - c.Next() - return - } - - var productName string - poetryClient := poetry.New(config.PoetryServiceAddress(), config.PoetryAPIRootKey()) - - if paramType == permission.ParamType { - productName = c.Param("name") - if productName == "" { - productName = c.Param("productName") - } - } else if paramType == permission.QueryType { - productName = c.Query("productName") - if productName == "" { - productName = c.Query("productTmpl") - } - } else if paramType == permission.ContextKeyType { - productName = c.GetString("productName") - } - - if productName == "" { - log.Errorf("productName is null") - c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"message": "Authentication failed."}) - return - } - - //其他权限 - permissionUUID := "" - if len(permissionUUIDs) == 1 { - permissionUUID = permissionUUIDs[0] - } else if len(permissionUUIDs) == 2 || len(permissionUUIDs) == 3 { - envType := c.Query("envType") - switch envType { - case setting.ProdENV: - permissionUUID = permissionUUIDs[1] - default: - permissionUUID = permissionUUIDs[0] - } - } - - //判断用户是否有操作的权限uuid - if rolePermissionMap, err := poetryClient.GetUserPermissionUUIDMap(productName, permissionUUID, user.ID, log); err == nil { - if rolePermissionMap["isContain"] { - c.Next() - return - } - } else { - c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"message": "Authentication failed."}) - return - } - - //判断用户是否有环境授权权限uuid - if roleEnvPermissions, err := poetryClient.ListEnvRolePermission(productName, "", 0, log); err == nil { - for _, roleEnvPermission := range roleEnvPermissions { - if roleEnvPermission.PermissionUUID == permissionUUID { - c.Next() - return - } - } - } else { - c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"message": "Authentication failed."}) - return - } - - if len(permissionUUIDs) == 3 { - permissionUUID = permissionUUIDs[2] - if rolePermissionMap, err := poetryClient.GetUserPermissionUUIDMap(productName, permissionUUID, user.ID, log); err == nil { - if rolePermissionMap["isContain"] { - c.Next() - return - } - } else { - c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"message": "Authentication failed."}) - return - } - } - - // 判断项目里面是否有all-users设置，以及该有的权限 - productRole, _ := poetryClient.ListRoles(productName, log) - if productRole != nil { - uuids, err := poetryClient.GetUserPermissionUUIDs(productRole.ID, productName, log) - if err != nil { - log.Errorf("GetUserPermissionUUIDs error: %v", err) - c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"message": "Authentication failed."}) - return - } - if sets.NewString(uuids...).Has(permissionUUID) { - c.Next() - return - } - } - - //判断普通用户的默认权限是否包含 - uuids, err := poetryClient.GetUserPermissionUUIDs(setting.RoleUserID, "", log) - if err != nil { - log.Errorf("GetUserPermissionUUIDs error: %v", err) - c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"message": "Authentication failed."}) - return - } - if sets.NewString(uuids...).Has(permissionUUID) { - c.Next() - return - } - - c.AbortWithStatusJSON(http.StatusForbidden, gin.H{"message": "对不起,您没有权限访问!"}) - } -} - -func authUser(username interface{}, c *gin.Context) (*permission.User, error) { - - username, ok := username.(string) - if !ok || username == "" { - return nil, fmt.Errorf("empty user id") - } - - userInfo, isExist := c.Get(setting.SessionUser) - user := new(poetry.UserInfo) - if isExist { - user = userInfo.(*poetry.UserInfo) - } - //说明用户可能已经删除 - if user.Name == "" { - return nil, fmt.Errorf("user %s has been disabled", username) - } - - return poetry.ConvertUserInfo(user), nil -} diff --git a/pkg/middleware/gin/reponse.go b/pkg/middleware/gin/response.go similarity index 100% rename from pkg/middleware/gin/reponse.go rename to pkg/middleware/gin/response.go diff --git a/pkg/setting/consts.go b/pkg/setting/consts.go index 52eb074ae75a38f9db723b405a7e0e047bcc7d22..f3eca36922b223c709b7ad8b9155c017c2ef8bf5 100644 --- a/pkg/setting/consts.go +++ b/pkg/setting/consts.go @@ -16,6 +16,8 @@ limitations under the License. package setting +import "time" + const LocalConfig = "local.env" // envs @@ -28,7 +30,11 @@ const ( ENVMongoDBConnectionString = "MONGODB_CONNECTION_STRING" ENVAslanDBName = "ASLAN_DB" ENVHubAgentImage = "HUB_AGENT_IMAGE" - ENVPoetryAPIRootKey = "POETRY_API_ROOT_KEY" + ENVResourceServerImage = "RESOURCE_SERVER_IMAGE" + ENVMysqlUser = "MYSQL_USER" + ENVMysqlPassword = "MYSQL_PASSWORD" + ENVMysqlHost = "MYSQL_HOST" + ENVMysqlUserDb = "MYSQL_USER_DB" // Aslan ENVPodName = "BE_POD_NAME" @@ -48,6 +54,7 @@ const ( ENVReaperImage = "REAPER_IMAGE" ENVReaperBinaryFile = "REAPER_BINARY_FILE" ENVPredatorImage = "PREDATOR_IMAGE" + EnvPackagerImage = "PACKAGER_IMAGE" ENVDockerHosts = "DOCKER_HOSTS" @@ -76,13 +83,11 @@ const ( KubernetesServicePort = "KUBERNETES_SERVICE_PORT" Token = "X-API-Tunnel-Token" Params = "X-API-Tunnel-Params" + AslanBaseAddr = "ASLAN_BASE_ADDR" // warpdrive WarpDrivePodName = "WD_POD_NAME" ReleaseImageTimeout = "RELEASE_IMAGE_TIMEOUT" - DefaultRegistryAddr = "DEFAULT_REG_ADDRESS" - DefaultRegistryAK = "DEFAULT_REG_ACCESS_KEY" - DefaultRegistrySK = "DEFAULT_REG_SECRET_KEY" // reaper Home = "HOME" @@ -96,28 +101,48 @@ const ( // jenkins JenkinsBuildImage = "JENKINS_BUILD_IMAGE" + // dind + DindImage = "DIND_IMAGE" + DebugMode = "debug" ReleaseMode = "release" TestMode = "test" + + // user + ENVIssuerURL = "ISSUER_URL" + ENVClientID = "CLIENT_ID" + ENVClientSecret = "CLIENT_SECRET" + ENVRedirectURI = "REDIRECT_URI" + ENVSecretKey = "SECRET_KEY" + ENVMysqlUserDB = "MYSQL_USER_DB" + ENVScopes = "SCOPES" + ENVTokenExpiresAt = "TOKEN_EXPIRES_AT" + ENVUserPort = "USER_PORT" + + // initconfig + ENVAdminEmail = "ADMIN_EMAIL" + ENVAdminPassword = "ADMIN_PASSWORD" + PresetAccount = "admin" ) // k8s concepts const ( - Secret = "Secret" - ConfigMap = "ConfigMap" - Ingress = "Ingress" - Service = "Service" - Deployment = "Deployment" - StatefulSet = "StatefulSet" - Pod = "Pod" - ReplicaSet = "ReplicaSet" - Job = "Job" - CronJob = "CronJob" - ClusterRoleBinding = "ClusterRoleBinding" - ServiceAccount = "ServiceAccount" - ClusterRole = "ClusterRole" - Role = "Role" - RoleBinding = "RoleBinding" + Secret = "Secret" + ConfigMap = "ConfigMap" + Ingress = "Ingress" + PersistentVolumeClaim = "PersistentVolumeClaim" + Service = "Service" + Deployment = "Deployment" + StatefulSet = "StatefulSet" + Pod = "Pod" + ReplicaSet = "ReplicaSet" + Job = "Job" + CronJob = "CronJob" + ClusterRoleBinding = "ClusterRoleBinding" + ServiceAccount = "ServiceAccount" + ClusterRole = "ClusterRole" + Role = "Role" + RoleBinding = "RoleBinding" // labels TaskLabel = "s-task" @@ -145,6 +170,10 @@ const ( EditorIDAnnotation = companyLabel + "/" + "editor-id" LastUpdateTimeAnnotation = companyLabel + "/" + "last-update-time" + JobLabelTaskKey = "s-task" + JobLabelNameKey = "s-name" + JobLabelSTypeKey = "s-type" + LabelValueTrue = "true" // Pod status @@ -169,17 +198,21 @@ const ( // annotations HelmReleaseNameAnnotation = "meta.helm.sh/release-name" - EnvCreatedBy = "createdBy" - EnvCreator = "koderover" - PodReady = "ready" - JobReady = "Completed" - PodNotReady = "not ready" - HelmReleaseStatusPending = "Pending" - HelmReleaseStatusDeployed = "Deployed" + EnvCreatedBy = "createdBy" + EnvCreator = "koderover" + PodReady = "ready" + JobReady = "Completed" + PodNotReady = "not ready" APIVersionAppsV1 = "apps/v1" + RSASecretName = "zadig-rsa-key" + DefaultImagePullSecret = "default-registry-secret" + + PolicyMetaConfigMapName = "zadig-policy-meta" + PolicyURLConfigMapName = "zadig-policy-url" + PolicyRoleConfigMapName = "zadig-policy-role" ) const ( @@ -192,41 +225,49 @@ const ( DefaultIngressClass = "koderover-nginx" - // K8SDeployType 容器化部署方式 + // K8SDeployType Containerized Deployment K8SDeployType = "k8s" - // helm 部署 + // helm deployment HelmDeployType = "helm" - // PMDeployType physical machine deploy 脚本物理机部署方式 - PMDeployType = "pm" + // PMDeployType physical machine deploy method + PMDeployType = "pm" + TrusteeshipDeployType = "trusteeship" - // 基础设施 k8s类型 + // Infrastructure k8s type BasicFacilityK8S = "kubernetes" - // 基础设施 云主机 + // Infrastructure Cloud Hosting BasicFacilityCVM = "cloud_host" - // SourceFromZadig 配置来源由平台管理 + // SourceFromZadig Configuration sources are managed by the platform SourceFromZadig = "spock" - // SourceFromGitlab 配置来源为gitlab + // SourceFromGitlab The configuration source is gitlab SourceFromGitlab = "gitlab" - // SourceFromGithub 配置来源为github + // SourceFromGithub The configuration source is github SourceFromGithub = "github" - // SourceFromGitlab 配置来源为gerrit + // SourceFromGitlab The configuration source is gerrit SourceFromGerrit = "gerrit" - // SourceFromCodeHub 配置来源为codehub + // SourceFromCodeHub The configuration source is codehub SourceFromCodeHub = "codehub" - // SourceFromIlyshin 配置来源为ilyshin - SourceFromIlyshin = "ilyshin" - // SourceFromChartTemplate 配置来源为helmTemplate + // SourceFromGitee Configure the source as gitee + SourceFromGitee = "gitee" + // SourceFromGitee Configure the source as other + SourceFromOther = "other" + // SourceFromChartTemplate The configuration source is helmTemplate SourceFromChartTemplate = "chartTemplate" - // SourceFromPublicRepo 配置来源为publicRepo + // SourceFromPublicRepo The configuration source is publicRepo SourceFromPublicRepo = "publicRepo" + SourceFromChartRepo = "chartRepo" - // SourceFromGUI 配置来源为gui + // SourceFromGUI The configuration source is gui SourceFromGUI = "gui" //SourceFromHelm SourceFromHelm = "helm" //SourceFromExternal SourceFromExternal = "external" + // service from yaml template + ServiceSourceTemplate = "template" + SourceFromPM = "pm" + SourceFromGitRepo = "repo" ProdENV = "prod" TestENV = "test" @@ -238,7 +279,7 @@ const ( PublicService = "public" - // onboarding流程第二步 + // The second step of the onboarding process OnboardingStatusSecond = 2 Unset = "UNSET" @@ -254,11 +295,16 @@ const ( ) const ( - SessionUsername = "Username" - SessionUser = "User" - UserAPIKey = "X-API-KEY" - RootAPIKey = "X-ROOT-API-KEY" - TIMERAPIKEY = "X-TIMER-API-KEY" + DeliveryVersionTypeChart = "HelmChart" + DeliveryVersionTypeK8SWorkflow = "K8SWorkflow" +) + +const ( + DeliveryDeployTypeImage = "image" + DeliveryDeployTypeChart = "chart" +) + +const ( AuthorizationHeader = "Authorization" ) @@ -286,16 +332,23 @@ const ( ReleaseImageJob = "docker-release" ) +const ( + BuildChartPackage = "chart-package" +) + const ( JenkinsBuildJob = "jenkins-build" ) // counter prefix const ( - PipelineTaskFmt = "PipelineTask:%s" - WorkflowTaskFmt = "WorkflowTask:%s" - TestTaskFmt = "TestTask:%s" - ServiceTaskFmt = "ServiceTask:%s" + PipelineTaskFmt = "PipelineTask:%s" + WorkflowTaskFmt = "WorkflowTask:%s" + WorkflowTaskV3Fmt = "WorkflowTaskV3:%s" + WorkflowTaskV4Fmt = "WorkflowTaskV4:%s" + TestTaskFmt = "TestTask:%s" + ServiceTaskFmt = "ServiceTask:%s" + ScanningTaskFmt = "ScanningTask:%s" ) // Product Status @@ -309,22 +362,27 @@ const ( ProductStatusUnstable = "Unstable" ) +// DeliveryVersion status const ( - NormalModeProduct = "normal" + DeliveryVersionStatusSuccess = "success" + DeliveryVersionStatusFailed = "failed" + DeliveryVersionStatusCreating = "creating" + DeliveryVersionStatusRetrying = "retrying" ) -// roles const ( - RoleOwnerID = 3 - RoleUserID = 4 + DeliveryVersionPackageStatusSuccess = "success" + DeliveryVersionPackageStatusFailed = "failed" + DeliveryVersionPackageStatusWaiting = "waiting" + DeliveryVersionPackageStatusUploading = "uploading" +) - RoleUser = "user" // 普通用户 - RoleOwner = "owner" // 项目管理员 - RoleAdmin = "admin" // 超级管理员 - RoleContributor = "contributor" //开源项目贡献者 - SystemUser = "system" +const ( + NormalModeProduct = "normal" +) - GuestAccount = "guest2019" +const ( + SystemUser = "system" ) // events @@ -468,9 +526,6 @@ const ( // helm related const ( - ValuesYamlSourceFreeEdit = "freeEdit" - ValuesYamlSourceGitRepo = "gitRepo" - // components used to search image paths from yaml PathSearchComponentRepo = "repo" PathSearchComponentImage = "image" @@ -482,13 +537,125 @@ const ( AliyunHost = ".aliyuncs.com" ) +// Dockerfile parsing consts +const ( + DockerfileCmdArg = "ARG" +) + +// Dockerfile template constant +const ( + DockerfileSourceLocal = "local" + DockerfileSourceTemplate = "template" + + ZadigDockerfilePath = "zadig-dockerfile" +) + +// Yaml template constant +const ( + RegExpParameter = `{{.(\w)+}}` +) + +// template common constant +const ( + TemplateVariableProduct = "$T-Project$" + TemplateVariableProductDescription = "项目名称" + TemplateVariableService = "$T-Service$" + TemplateVariableServiceDescription = "服务名称" +) + const MaxTries = 1 const DogFood = "/var/run/koderover-dog-food" +const ProgressFile = "/var/log/job-progress" + const ( ResponseError = "error" ResponseData = "response" ) const ChartTemplatesPath = "charts" + +type RoleType string + +const ( + Contributor RoleType = "contributor" + ReadOnly RoleType = "read-only" + ProjectAdmin RoleType = "project-admin" + SystemAdmin RoleType = "admin" + ReadProjectOnly RoleType = "read-project-only" +) + +// ModernWorkflowType 自由编排工作流 +const ModernWorkflowType = "ModernWorkflow" + +const ( + Subresource = "subresource" + StatusSubresource = "status" + IngressSubresource = "ingress" + ResourcesHeader = "Resources" +) + +type K8SClusterStatus string + +const ( + Disconnected K8SClusterStatus = "disconnected" + Pending K8SClusterStatus = "pending" + Normal K8SClusterStatus = "normal" + Abnormal K8SClusterStatus = "abnormal" +) + +type PMHostStatus string + +const ( + PMHostStatusNormal PMHostStatus = "normal" + PMHostStatusAbnormal PMHostStatus = "abnormal" +) + +const PMHostDefaultPort int64 = 22 + +type ResetImagePolicyType string + +const ( + ResetImagePolicyTaskCompleted ResetImagePolicyType = "taskCompleted" + ResetImagePolicyTaskCompletedOrder ResetImagePolicyType = "" + ResetImagePolicyDeployFailed ResetImagePolicyType = "deployFailed" + ResetImagePolicyTestFailed ResetImagePolicyType = "testFailed" +) + +const LocalClusterID = "0123456789abcdef12345678" + +const DefaultLoginLocal = "local" + +const RequestModeOpenAPI = "openAPI" + +const DeployTimeout = 60 * 10 // 10 minutes + +const UpdateEnvTimeout = 60 * 5 * time.Second + +// list namespace type +const ( + ListNamespaceTypeCreate = "create" +) + +const ( + InformerNamingConvention = "%s-%s" +) + +type ResourceType string + +const ( + ResourceTypeSystem ResourceType = "system" + ResourceTypeCustom ResourceType = "custom" +) + +// AttachedClusterNamespace is the namespace Zadig uses in attached cluster. +// Note: **Restricted because of product design since v1.9.0**. +const AttachedClusterNamespace = "koderover-agent" + +const ArtifactResultOut = "artifactResultOut.tar.gz" + +const ( + DefaultReleaseNaming = "$Service$" + ReleaseNamingPlaceholder = "$Namespace$-$Service$" +) diff --git a/pkg/setting/types.go b/pkg/setting/types.go index dd56e0cffdc9adb805e8d6ad9c04143a15dadb82..a02b516c882b2d84bd3a0bcca71c48239cf34a7b 100644 --- a/pkg/setting/types.go +++ b/pkg/setting/types.go @@ -29,6 +29,62 @@ const ( LowRequest Request = "low" // MinRequest 2 CPU 2 G MinRequest Request = "min" + // DefaultRequest 4 CPU 8 G + DefaultRequest Request = "default" + // DefineRequest x CPU x G + DefineRequest Request = "define" +) + +type RequestSpec struct { + CpuLimit int `bson:"cpu_limit" json:"cpu_limit" yaml:"cpu_limit"` + MemoryLimit int `bson:"memory_limit" json:"memory_limit" yaml:"memory_limit"` + CpuReq int `bson:"cpu_req,omitempty" json:"cpu_req,omitempty" yaml:"cpu_req,omitempty"` + MemoryReq int `bson:"memory_req,omitempty" json:"memory_req,omitempty" yaml:"memory_req,omitempty"` +} + +var ( + // HighRequestSpec 16 CPU 32 G + HighRequestSpec = RequestSpec{ + CpuLimit: 16000, + MemoryLimit: 32768, + CpuReq: 4000, + MemoryReq: 4096, + } + // MediumRequestSpec 8 CPU 16 G + MediumRequestSpec = RequestSpec{ + CpuLimit: 8000, + MemoryLimit: 16384, + CpuReq: 2000, + MemoryReq: 2048, + } + // LowRequestSpec 4 CPU 8 G + LowRequestSpec = RequestSpec{ + CpuLimit: 4000, + MemoryLimit: 8192, + CpuReq: 1000, + MemoryReq: 1024, + } + // MinRequestSpec 2 CPU 2 G + MinRequestSpec = RequestSpec{ + CpuLimit: 2000, + MemoryLimit: 2048, + CpuReq: 500, + MemoryReq: 512, + } + // DefineRequest X CPU X G + DefineRequestSpec = RequestSpec{} + DefaultRequestSpec = RequestSpec{ + CpuLimit: 4000, + MemoryLimit: 8192, + CpuReq: 1000, + MemoryReq: 1024, + } +) + +const ( + CpuUintM string = "m" + MemoryUintMi string = "Mi" + MemoryUintGi string = "Gi" ) var ValidName = regexp.MustCompile(`^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$`) @@ -42,11 +98,14 @@ const ( Cron // 5 HubServer // 6 PodExec // 7 - Poetry // 8 SonarQube // 9 WarpDrive // 10 Minio // 11 OPA // 12 + Policy // 13 + Config // 14 + User // 15 + Vendor ) type ServiceInfo struct { @@ -83,10 +142,6 @@ var Services = map[int]*ServiceInfo{ Name: "podexec", Port: 27000, }, - Poetry: { - Name: "poetry", - Port: 34001, - }, SonarQube: { Name: "sonarqube", Port: 80, @@ -103,4 +158,20 @@ var Services = map[int]*ServiceInfo{ Name: "opa", Port: 8181, }, + Policy: { + Name: "policy", + Port: 80, + }, + Config: { + Name: "config", + Port: 80, + }, + User: { + Name: "user", + Port: 80, + }, + Vendor: { + Name: "plutus-vendor", + Port: 29000, + }, } diff --git a/pkg/shared/client/aslan/client.go b/pkg/shared/client/aslan/client.go index c34553ca0b1af38144523f301ac94fc8853a2978..1732af27ddbe5d857ea649d418281a00a0eb30e4 100644 --- a/pkg/shared/client/aslan/client.go +++ b/pkg/shared/client/aslan/client.go @@ -17,43 +17,42 @@ limitations under the License. package aslan import ( - "github.com/koderover/zadig/pkg/setting" + "crypto/tls" + "github.com/koderover/zadig/pkg/tool/httpclient" ) type Client struct { *httpclient.Client - host string - token string + host string + token string external bool } -func New(host, token string) *Client { +func New(host string) *Client { c := httpclient.New( - httpclient.SetAuthScheme(setting.RootAPIKey), - httpclient.SetAuthToken(token), - httpclient.SetHostURL(host+"/api"), + httpclient.SetHostURL(host + "/api"), ) return &Client{ Client: c, host: host, - token: token, } } func NewExternal(host, token string) *Client { c := httpclient.New( - httpclient.SetAuthScheme(setting.UserAPIKey), httpclient.SetAuthToken(token), httpclient.SetHostURL(host+"/api/aslan"), ) + c.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}) + return &Client{ - Client: c, - host: host, - token: token, + Client: c, + host: host, + token: token, external: true, } } diff --git a/pkg/shared/client/aslan/environment.go b/pkg/shared/client/aslan/environment.go index 1a82a5db02005bc5df6d595871651cea4a020cb1..24794dd20d6745ef1cb50db9a7bb17a1c09e7e41 100644 --- a/pkg/shared/client/aslan/environment.go +++ b/pkg/shared/client/aslan/environment.go @@ -21,13 +21,14 @@ import ( "github.com/koderover/zadig/pkg/tool/httpclient" "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types" ) func (c *Client) ListEnvironments(projectName string) ([]*Environment, error) { url := "/environment/environments" res := make([]*Environment, 0) - _, err := c.Get(url, httpclient.SetQueryParam("productName", projectName), httpclient.SetResult(&res)) + _, err := c.Get(url, httpclient.SetQueryParam("projectName", projectName), httpclient.SetResult(&res)) if err != nil { return nil, err } @@ -36,10 +37,10 @@ func (c *Client) ListEnvironments(projectName string) ([]*Environment, error) { } func (c *Client) GetEnvironment(envName, projectName string) (*Environment, error) { - url := fmt.Sprintf("/environment/environments/%s", projectName) + url := fmt.Sprintf("/environment/environments/%s", envName) res := &Environment{} - _, err := c.Get(url, httpclient.SetQueryParam("envName", envName), httpclient.SetResult(res)) + _, err := c.Get(url, httpclient.SetQueryParam("projectName", projectName), httpclient.SetResult(res)) if err != nil { return nil, err } @@ -48,10 +49,10 @@ func (c *Client) GetEnvironment(envName, projectName string) (*Environment, erro } func (c *Client) ListHelmServicesInEnvironment(envName, projectName string) ([]*Service, error) { - url := fmt.Sprintf("/environment/environments/%s/groups/helm", projectName) + url := fmt.Sprintf("/environment/environments/%s/groups", envName) res := &ServicesResp{} - _, err := c.Get(url, httpclient.SetQueryParam("envName", envName), httpclient.SetResult(res)) + _, err := c.Get(url, httpclient.SetQueryParam("projectName", projectName), httpclient.SetResult(res)) if err != nil { return nil, err } @@ -60,20 +61,20 @@ func (c *Client) ListHelmServicesInEnvironment(envName, projectName string) ([]* } func (c *Client) ListServices(envName, projectName string) ([]*Service, error) { - url := fmt.Sprintf("/environment/environments/%s/groups", projectName) + url := fmt.Sprintf("/environment/environments/%s/groups", envName) res := make([]*Service, 0) - _, err := c.Get(url, httpclient.SetQueryParam("envName", envName), httpclient.SetResult(&res)) + _, err := c.Get(url, httpclient.SetQueryParam("projectName", projectName), httpclient.SetResult(&res)) return res, err } func (c *Client) GetServiceDetail(projectName, serviceName, envName string) (*ServiceDetail, error) { - url := fmt.Sprintf("/environment/environments/%s/services/%s", projectName, serviceName) + url := fmt.Sprintf("/environment/environments/%s/services/%s", envName, serviceName) res := &ServiceDetail{} req := map[string]string{ - "envName": envName, + "projectName": projectName, } _, err := c.Get(url, httpclient.SetQueryParams(req), httpclient.SetResult(res)) if err != nil { @@ -100,7 +101,7 @@ func (c *Client) ListServicesStatusByEnvironment(envName, projectName string) ([ for _, s := range ss { res = append(res, &ServiceStatus{ ServiceName: s.ServiceName, - Status: s.Status, + Status: s.Status, }) } @@ -113,10 +114,29 @@ func (c *Client) ListServicesStatusByEnvironment(envName, projectName string) ([ for _, s := range ss { res = append(res, &ServiceStatus{ ServiceName: s.ServiceName, - Status: s.Status, + Status: s.Status, }) } } return res, nil } + +func (c *Client) PatchWorkload(projectName, envName, serviceName, devImage string) (*types.WorkloadInfo, error) { + res := &types.WorkloadInfo{} + + body := types.StartDevmodeInfo{ + DevImage: devImage, + } + url := fmt.Sprintf("/environment/environments/%s/services/%s/devmode/patch?projectName=%s", envName, serviceName, projectName) + _, err := c.Post(url, httpclient.SetBody(body), httpclient.SetResult(res)) + + return res, err +} + +func (c *Client) RecoverWorkload(projectName, envName, serviceName string) error { + url := fmt.Sprintf("/environment/environments/%s/services/%s/devmode/recover?projectName=%s", envName, serviceName, projectName) + _, err := c.Post(url) + + return err +} diff --git a/pkg/shared/client/aslan/healthz.go b/pkg/shared/client/aslan/healthz.go new file mode 100644 index 0000000000000000000000000000000000000000..d47bcee3b959bb3d590ca8e46d9328540a174a4e --- /dev/null +++ b/pkg/shared/client/aslan/healthz.go @@ -0,0 +1,24 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aslan + +// Healthz api/health +func (c *Client) Healthz() error { + url := "/health" + _, err := c.Get(url) + return err +} diff --git a/pkg/shared/client/aslan/login.go b/pkg/shared/client/aslan/login.go new file mode 100644 index 0000000000000000000000000000000000000000..7f2cc5ea88605e6282d494c6694323c1f5fc06a9 --- /dev/null +++ b/pkg/shared/client/aslan/login.go @@ -0,0 +1,39 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aslan + +import ( + "fmt" + + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +type GetDefaultLoginResp struct { + DefaultLogin string `json:"default_login"` +} + +func (c *Client) GetDefaultLogin() (*GetDefaultLoginResp, error) { + url := fmt.Sprintf("/system/login/default") + + res := &GetDefaultLoginResp{} + _, err := c.Get(url, httpclient.SetResult(res)) + if err != nil { + return nil, err + } + + return res, nil +} diff --git a/pkg/shared/client/aslan/audit_log.go b/pkg/shared/client/aslan/multicluster.go similarity index 34% rename from pkg/shared/client/aslan/audit_log.go rename to pkg/shared/client/aslan/multicluster.go index f41cab616cae7efba83c29946bc80a2f64ff7411..0611bcd85d8a69a0c827ce9a6008ac47fb51cf76 100644 --- a/pkg/shared/client/aslan/audit_log.go +++ b/pkg/shared/client/aslan/multicluster.go @@ -17,67 +17,62 @@ limitations under the License. package aslan import ( + "encoding/json" "fmt" "time" - "go.uber.org/zap" - + "github.com/koderover/zadig/pkg/setting" "github.com/koderover/zadig/pkg/tool/httpclient" ) -type operationLog struct { - Username string `json:"username"` - ProductName string `json:"product_name"` - Method string `json:"method"` - Function string `json:"function"` - Name string `json:"name"` - RequestBody string `json:"request_body"` - Status int `json:"status"` - CreatedAt int64 `json:"created_at"` -} - -type updateOperationArgs struct { - Status int `json:"status"` +type cluster struct { + ID string `json:"id,omitempty"` + Name string `json:"name"` + Status setting.K8SClusterStatus `json:"status"` + Local bool `json:"local"` } -type AddAuditLogResp struct { - OperationLogID string `json:"id"` -} - -func (c *Client) AddAuditLog(username, productName, method, function, detail, requestBody string, log *zap.SugaredLogger) (string, error) { - url := "/system/operation" - req := operationLog{ - Username: username, - ProductName: productName, - Method: method, - Function: function, - Name: detail, - RequestBody: requestBody, - Status: 0, - CreatedAt: time.Now().Unix(), +func (c *Client) AddLocalCluster() error { + url := "/cluster/clusters" + req := cluster{ + ID: setting.LocalClusterID, + Name: fmt.Sprintf("%s-%s", "local", time.Now().Format("20060102150405")), } - var operationLogID AddAuditLogResp - _, err := c.Post(url, httpclient.SetBody(req), httpclient.SetResult(&operationLogID)) + _, err := c.Post(url, httpclient.SetBody(req)) if err != nil { - log.Errorf("Failed to add audit log, error: %s", err) - return "", err + return fmt.Errorf("Failed to add multi cluster, error: %s", err) } - return operationLogID.OperationLogID, nil + return nil +} + +type clusterResp struct { + Name string `json:"name"` + Local bool `json:"local"` } -func (c *Client) UpdateAuditLog(id string, status int, log *zap.SugaredLogger) error { - url := fmt.Sprintf("/system/operation/%s", id) - req := updateOperationArgs{ - Status: status, - } +type ErrorMessage struct { + Type string `json:"type"` + Code int `json:"code"` +} + +func (c *Client) GetLocalCluster() (*clusterResp, error) { + url := fmt.Sprintf("/cluster/clusters/%s", setting.LocalClusterID) - _, err := c.Put(url, httpclient.SetBody(req)) + clusterResp := &clusterResp{} + resp, err := c.Get(url, httpclient.SetResult(clusterResp)) if err != nil { - log.Errorf("Failed to update audit log, error: %s", err) - return err + errorMessage := new(ErrorMessage) + err := json.Unmarshal(resp.Body(), errorMessage) + if err != nil { + return nil, fmt.Errorf("Failed to get cluster, error: %s", err) + } + if errorMessage.Code == 6643 { + return nil, nil + } + return nil, fmt.Errorf("Failed to get cluster, error: %s", err) } - return nil + return clusterResp, nil } diff --git a/pkg/shared/poetry/token.go b/pkg/shared/client/aslan/registries.go similarity index 62% rename from pkg/shared/poetry/token.go rename to pkg/shared/client/aslan/registries.go index bf31b540fcf2c5ddc14f8f6f973ce81b22b33add..24b1add5f95baff69310cdf1b00f637297bdf9bc 100644 --- a/pkg/shared/poetry/token.go +++ b/pkg/shared/client/aslan/registries.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The KodeRover Authors. +Copyright 2022 The KodeRover Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,23 +14,18 @@ See the License for the specific language governing permissions and limitations under the License. */ -package poetry +package aslan import "github.com/koderover/zadig/pkg/tool/httpclient" -type signature struct { - Token string `json:"token"` - Username string `json:"username"` -} - -func (c *Client) SignatureCheck(username, token string) error { - url := "/directory/token/check" +func (c *Client) ListRegistries() ([]*RegistryInfo, error) { + url := "/system/registry" + res := make([]*RegistryInfo, 0) - s := &signature{Username: username, Token: token} - _, err := c.Post(url, httpclient.SetBody(s)) + _, err := c.Get(url, httpclient.SetResult(&res)) if err != nil { - return err + return nil, err } - return nil + return res, nil } diff --git a/pkg/shared/poetry/auth.go b/pkg/shared/client/aslan/rsa_key.go similarity index 53% rename from pkg/shared/poetry/auth.go rename to pkg/shared/client/aslan/rsa_key.go index 201659581436fe6a3c99fe09bc7a558235f8d7a9..5211c9d8e4db9aed7ba1bb6baa9c3b0c93b5be6f 100644 --- a/pkg/shared/poetry/auth.go +++ b/pkg/shared/client/aslan/rsa_key.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The KodeRover Authors. +Copyright 2022 The KodeRover Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,30 +14,26 @@ See the License for the specific language governing permissions and limitations under the License. */ -package poetry +package aslan import ( "fmt" - "github.com/koderover/zadig/pkg/setting" "github.com/koderover/zadig/pkg/tool/httpclient" ) -type userView struct { - User *UserInfo `json:"info"` +type GetAesKeyFromEncryptedKeyResp struct { + PlainText string `json:"plain_text"` } -func GetUserDetailByToken(poetryHost, token string) (*UserInfo, error) { - url := "/directory/user/detail" +func (c *Client) GetTextFromEncryptedKey(encryptedKey string) (*GetAesKeyFromEncryptedKeyResp, error) { + url := fmt.Sprintf("/system/rsaKey/decryptedText?encryptedKey=%s", encryptedKey) - cl := httpclient.New(httpclient.SetHostURL(poetryHost)) - - //根据token获取用户 - userViewInfo := &userView{} - _, err := cl.Get(url, httpclient.SetHeader(setting.AuthorizationHeader, fmt.Sprintf("%s %s", setting.UserAPIKey, token)), httpclient.SetResult(userViewInfo)) + res := &GetAesKeyFromEncryptedKeyResp{} + _, err := c.Get(url, httpclient.SetQueryParam("encryptedKey", encryptedKey), httpclient.SetResult(res)) if err != nil { return nil, err } - return userViewInfo.User, nil + return res, nil } diff --git a/pkg/shared/client/aslan/system.go b/pkg/shared/client/aslan/system.go new file mode 100644 index 0000000000000000000000000000000000000000..2d3c5590920c9919e6e078f2e6d82aeb619e6943 --- /dev/null +++ b/pkg/shared/client/aslan/system.go @@ -0,0 +1,45 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aslan + +import ( + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +type CleanConfig struct { + Cron string `json:"cron"` + CronEnabled bool `json:"cron_enabled"` +} + +func (c *Client) GetDockerCleanConfig() (*CleanConfig, error) { + url := "/system/cleanCache/state" + + res := new(CleanConfig) + + _, err := c.Get(url, httpclient.SetResult(res)) + if err != nil { + return nil, err + } + + return res, nil +} + +func (c *Client) DockerClean() error { + url := "/system/cleanCache/oneClick" + _, err := c.Post(url) + return err +} diff --git a/pkg/shared/client/aslan/types.go b/pkg/shared/client/aslan/types.go index a3bcdc3c040b2f2be65a6679d84df4323d125489..47d041072fb865c9f0b5df7c672efc4a5717ffc5 100644 --- a/pkg/shared/client/aslan/types.go +++ b/pkg/shared/client/aslan/types.go @@ -16,6 +16,8 @@ limitations under the License. package aslan +import "go.mongodb.org/mongo-driver/bson/primitive" + type Environment struct { ID string `json:"id"` ProductName string `json:"product_name"` @@ -111,3 +113,18 @@ type ServiceStatus struct { ServiceName string `json:"service_name"` Status string `json:"status"` } + +type RegistryInfo struct { + ID primitive.ObjectID `json:"id,omitempty"` + RegAddr string `json:"reg_addr"` + AdvancedSetting *RegistryAdvancedSetting `json:"advanced_setting"` +} + +type RegistryAdvancedSetting struct { + // indicator to make sure if the advanced has been modified + Modified bool `json:"modified"` + // New field since v1.11 to support self-signed TLS certificate + TLSEnabled bool `json:"enable_tls"` + TLSCert string `json:"tls_cert"` + TLSKey string `json:"tls_key"` +} diff --git a/pkg/shared/client/aslan/workflow.go b/pkg/shared/client/aslan/workflow.go new file mode 100644 index 0000000000000000000000000000000000000000..b7812e6cbe1337a24b29762924b45b9679a1138f --- /dev/null +++ b/pkg/shared/client/aslan/workflow.go @@ -0,0 +1,44 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aslan + +import ( + "encoding/json" + "fmt" + + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +type workflowConcurrencySettingResp struct { + WorkflowConcurrency int64 `json:"workflow_concurrency"` + BuildConcurrency int64 `json:"build_concurrency"` +} + +func (c *Client) GetWorkflowConcurrencySetting() (*workflowConcurrencySettingResp, error) { + url := "/system/concurrency/workflow" + + resp := &workflowConcurrencySettingResp{} + res, err := c.Get(url, httpclient.SetResult(resp)) + if err != nil { + errorMessage := new(ErrorMessage) + err := json.Unmarshal(res.Body(), errorMessage) + if err != nil { + return nil, fmt.Errorf("failed to get workflow concurrency settings, error: %s", err) + } + } + return resp, nil +} diff --git a/pkg/shared/client/aslanx/client.go b/pkg/shared/client/aslanx/client.go index 05caf54c1d2bb3ef4c910a9097f6b2538e26d89f..0408946276a8e421077c58040d85f9a80b7e45bf 100644 --- a/pkg/shared/client/aslanx/client.go +++ b/pkg/shared/client/aslanx/client.go @@ -17,27 +17,22 @@ limitations under the License. package aslanx import ( - "github.com/koderover/zadig/pkg/setting" "github.com/koderover/zadig/pkg/tool/httpclient" ) type Client struct { *httpclient.Client - host string - token string + host string } -func New(host, token string) *Client { +func New(host string) *Client { c := httpclient.New( - httpclient.SetAuthScheme(setting.RootAPIKey), - httpclient.SetAuthToken(token), httpclient.SetHostURL(host), ) return &Client{ Client: c, host: host, - token: token, } } diff --git a/pkg/shared/client/label/client.go b/pkg/shared/client/label/client.go new file mode 100644 index 0000000000000000000000000000000000000000..e350ed680b22539bb7bbf5703946fe4b6919eb11 --- /dev/null +++ b/pkg/shared/client/label/client.go @@ -0,0 +1,40 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package label + +import ( + "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +type Client struct { + *httpclient.Client + + host string +} + +func New() *Client { + host := config.AslanServiceAddress() + c := httpclient.New( + httpclient.SetHostURL(host + "/api"), + ) + + return &Client{ + Client: c, + host: host, + } +} diff --git a/pkg/shared/client/label/label.go b/pkg/shared/client/label/label.go new file mode 100644 index 0000000000000000000000000000000000000000..091b05909573334cb14e822825450ef7919ec097 --- /dev/null +++ b/pkg/shared/client/label/label.go @@ -0,0 +1,133 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package label + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" + + "github.com/koderover/zadig/pkg/tool/httpclient" + "github.com/koderover/zadig/pkg/tool/log" +) + +type Label struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + Type string `bson:"type" json:"type"` + Key string `bson:"key" json:"key"` + Value string `bson:"value" json:"value"` + CreateBy string `bson:"create_by" json:"create_by"` + CreateTime int64 `bson:"create_time" json:"create_time"` +} + +type ListResourcesByLabelsReq struct { + LabelFilters []Label `json:"label_filters"` +} + +type Resource struct { + Name string `json:"name"` + ProjectName string `json:"project_name"` + Type string `json:"type"` +} + +type ListResourcesByLabelsResp struct { + Resources map[string][]Resource `json:"resources"` +} + +func (c *Client) ListResourcesByLabels(request ListResourcesByLabelsReq) (*ListResourcesByLabelsResp, error) { + url := "/label/labels/resources-by-labels" + + var resources *ListResourcesByLabelsResp + _, err := c.Post(url, httpclient.SetBody(request), httpclient.SetResult(&resources)) + if err != nil { + log.Errorf("Failed to add audit log, error: %s", err) + return nil, err + } + + return resources, nil +} + +type LabelModel struct { + Key string `json:"key"` + Value string `json:"value"` +} + +type ListLabelsArgs struct { + Labels []LabelModel `json:"labels"` +} + +type ListLabelsResp struct { + Labels []Label `json:"labels"` +} + +func (c *Client) ListLabels(request ListLabelsArgs) (*ListLabelsResp, error) { + url := "/label/labels" + + var resources *ListLabelsResp + _, err := c.Post(url, httpclient.SetBody(request), httpclient.SetResult(&resources)) + if err != nil { + log.Errorf("Failed to ListLabels, error: %s", err) + return nil, err + } + + return resources, nil +} + +type ListLabelsByResourcesReq struct { + Resources []Resource `json:"resources"` +} + +type ListLabelsByResourcesResp struct { + Labels map[string][]*Label `json:"labels"` +} + +type CreateLabelsArgs struct { + Labels []Label `json:"labels"` +} + +func (c *Client) ListLabelsByResources(request ListLabelsByResourcesReq) (*ListLabelsByResourcesResp, error) { + url := "/label/labels/labels-by-resources" + + var resources *ListLabelsByResourcesResp + _, err := c.Post(url, httpclient.SetBody(request), httpclient.SetResult(&resources)) + if err != nil { + log.Errorf("Failed to ListLabelsByResources, error: %s", err) + return nil, err + } + + return resources, nil +} + +func (c *Client) CreateLabels(request CreateLabelsArgs) (*CreateLabelsResp, error) { + url := "/label/labels" + var resources *CreateLabelsResp + _, err := c.Post(url, httpclient.SetBody(request), httpclient.SetResult(&resources)) + if err != nil { + log.Errorf("Failed to CreateLabels, error: %s", err) + return nil, err + } + return resources, nil +} + +func (c *Client) DeleteLabels(ids []string) error { + url := "/label/labels?force=true" + _, err := c.Delete(url, httpclient.SetBody(ids)) + if err != nil { + log.Errorf("Failed to CreateLabels, error: %s", err) + return err + } + + return nil +} diff --git a/pkg/shared/client/label/label_binding.go b/pkg/shared/client/label/label_binding.go new file mode 100644 index 0000000000000000000000000000000000000000..3f0b7eb1c882a47f25ce6ac7faf3018fc5bd25c1 --- /dev/null +++ b/pkg/shared/client/label/label_binding.go @@ -0,0 +1,65 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package label + +import ( + "github.com/koderover/zadig/pkg/tool/httpclient" + "github.com/koderover/zadig/pkg/tool/log" +) + +type CreateLabelsResp struct { + LabelMap map[string]string `json:"label_map"` +} + +type CreateLabelBindingsArgs struct { + LabelBindings []LabelBinding `json:"label_bindings"` +} + +type LabelBinding struct { + Resource Resource `json:"resource"` + LabelID string `json:"label_id"` + CreateBy string `json:"create_by"` + CreateTime int64 `json:"create_time"` +} + +type DeleteLabelBindingsArgs struct { + LabelBindings []LabelBinding `json:"label_bindings"` +} + +func (c *Client) CreateLabelBindings(request CreateLabelBindingsArgs) error { + url := "/label/labelbindings" + + _, err := c.Post(url, httpclient.SetBody(request)) + if err != nil { + log.Errorf("Failed to CreateLabelBindings, error: %s", err) + return err + } + + return nil +} + +func (c *Client) DeleteLabelBindings(request DeleteLabelBindingsArgs) error { + url := "/label/labelbindings" + + _, err := c.Delete(url, httpclient.SetBody(request)) + if err != nil { + log.Errorf("Failed to DeleteLabelBindings, error: %s", err) + return err + } + + return nil +} diff --git a/pkg/shared/client/plutusvendor/client.go b/pkg/shared/client/plutusvendor/client.go new file mode 100644 index 0000000000000000000000000000000000000000..0fecc59cf1c7520fbabc54abd408c3f20c7fc501 --- /dev/null +++ b/pkg/shared/client/plutusvendor/client.go @@ -0,0 +1,41 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plutusvendor + +import ( + "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +type Client struct { + *httpclient.Client + + host string +} + +func New() *Client { + host := config.VendorServiceAddress() + + c := httpclient.New( + httpclient.SetHostURL(host + "/api/plutus"), + ) + + return &Client{ + Client: c, + host: host, + } +} diff --git a/pkg/shared/poetry/permission.go b/pkg/shared/client/plutusvendor/plutusvendor.go similarity index 62% rename from pkg/shared/poetry/permission.go rename to pkg/shared/client/plutusvendor/plutusvendor.go index c0f1d0793c17778fe2d49fac2520e7126b55eacc..1fccb841c9afd6ee217c058ba3b7ed7272268515 100644 --- a/pkg/shared/poetry/permission.go +++ b/pkg/shared/client/plutusvendor/plutusvendor.go @@ -14,23 +14,27 @@ See the License for the specific language governing permissions and limitations under the License. */ -package poetry +package plutusvendor import ( + "fmt" + "github.com/koderover/zadig/pkg/tool/httpclient" ) -type UserEnvPermission struct { - UserID int `json:"userId"` - ProductName string `json:"productName"` - EnvName string `json:"envName"` - PermissionUUIDs []string `json:"permissionUUIDs"` +type CheckSignatrueResp struct { + Code int64 `json:"code"` } -func (c *Client) CreateUserEnvPermission(p *UserEnvPermission) error { - url := "/directory/userEnvPermission" - _, err := c.Post(url, httpclient.SetBody(p)) +func (c *Client) CheckSignature(userNum int64) (*CheckSignatrueResp, error) { + url := fmt.Sprintf("/signature/check?user_num=%d", userNum) + res := &CheckSignatrueResp{} + _, err := c.Post(url, httpclient.SetResult(res)) + return res, err +} +func (c *Client) Health() error { + url := "/health" + _, err := c.Get(url) return err - } diff --git a/pkg/shared/client/policy/client.go b/pkg/shared/client/policy/client.go new file mode 100644 index 0000000000000000000000000000000000000000..34f60e2925dc7969c082f0c2c70fd4f23094b2d2 --- /dev/null +++ b/pkg/shared/client/policy/client.go @@ -0,0 +1,41 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package policy + +import ( + "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +type Client struct { + *httpclient.Client + + host string +} + +func NewDefault() *Client { + host := config.PolicyServiceAddress() + + c := httpclient.New( + httpclient.SetHostURL(host + "/api/v1"), + ) + + return &Client{ + Client: c, + host: host, + } +} diff --git a/pkg/shared/client/policy/policy.go b/pkg/shared/client/policy/policy.go new file mode 100644 index 0000000000000000000000000000000000000000..a27471466cb069500c9948b982702cad7e7766b6 --- /dev/null +++ b/pkg/shared/client/policy/policy.go @@ -0,0 +1,117 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package policy + +import ( + "fmt" + + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/httpclient" + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types" +) + +type CreatePoliciesArgs struct { + Policies []*types.Policy `json:"policies"` +} + +type DeletePoliciesArgs struct { + Names []string `json:"names"` +} + +type PolicyBinding struct { + Name string `json:"name"` + UID string `json:"uid"` + Policy string `json:"policy"` + Preset bool `json:"preset"` + Type setting.ResourceType `json:"type"` +} + +func (c *Client) CreatePolicyBinding(projectName string, policyBindings []*PolicyBinding) error { + url := fmt.Sprintf("/policybindings?projectName=%s", projectName) + _, err := c.Post(url, httpclient.SetBody(policyBindings)) + return err +} + +func (c *Client) CreatePolicies(ns string, request CreatePoliciesArgs) error { + url := fmt.Sprintf("/policies?projectName=%s", ns) + + _, err := c.Post(url, httpclient.SetBody(request)) + if err != nil { + log.Errorf("Failed to createPolicies, error: %s", err) + return err + } + + return nil +} + +func (c *Client) DeletePolicies(ns string, request DeletePoliciesArgs) error { + url := fmt.Sprintf("/policies/bulk-delete?projectName=%s", ns) + + _, err := c.Post(url, httpclient.SetBody(request)) + if err != nil { + log.Errorf("Failed to deletePolicies, error: %s", err) + return err + } + + return nil +} + +func (c *Client) DeletePolicyBindings(names []string, projectName string) error { + url := fmt.Sprintf("/policybindings/bulk-delete?projectName=%s", projectName) + nameArgs := &NameArgs{} + for _, v := range names { + nameArgs.Names = append(nameArgs.Names, v) + } + _, err := c.Post(url, httpclient.SetBody(nameArgs)) + return err +} + +func (c *Client) UpdatePolicy(ns string, policy *types.Policy) error { + url := fmt.Sprintf("/policies/%s?projectName=%s", policy.Name, ns) + _, err := c.Put(url, httpclient.SetBody(policy)) + return err +} + +type ResourcePermissionReq struct { + ProjectName string `json:"project_name" form:"project_name"` + Uid string `json:"uid" form:"uid"` + Resources []string `json:"resources" form:"resources"` + ResourceType string `json:"resource_type" form:"resource_type"` +} + +func (c *Client) GetResourcePermission(req *ResourcePermissionReq) (map[string][]string, error) { + url := fmt.Sprintf("/permission/resources") + result := make(map[string][]string) + _, err := c.Post(url, httpclient.SetResult(req), httpclient.SetResult(&result)) + if err != nil { + log.Errorf("Failed to get resourcePermission,err: %s", err) + return nil, err + } + return result, nil +} + +func (c *Client) GetPolicies(names string) ([]*types.Policy, error) { + url := fmt.Sprintf("/policies/bulk") + res := make([]*types.Policy, 0) + _, err := c.Get(url, httpclient.SetQueryParam("names", names), httpclient.SetResult(&res)) + if err != nil { + log.Errorf("Failed to getPolicies,err: %s", err) + return nil, err + } + return res, nil +} diff --git a/pkg/shared/client/policy/role.go b/pkg/shared/client/policy/role.go new file mode 100644 index 0000000000000000000000000000000000000000..27345de54e3038f87d648bb107be6110730a5558 --- /dev/null +++ b/pkg/shared/client/policy/role.go @@ -0,0 +1,122 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package policy + +import ( + "fmt" + + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/httpclient" + "github.com/koderover/zadig/pkg/types" +) + +type RoleBinding struct { + Name string `json:"name"` + UID string `json:"uid"` + Role string `json:"role"` + Preset bool `json:"preset"` + Type setting.ResourceType `json:"type"` +} + +func (c *Client) CreateOrUpdatePolicyRegistration(p *types.PolicyMeta) error { + url := fmt.Sprintf("/policymetas/%s", p.Resource) + + _, err := c.Put(url, httpclient.SetBody(p)) + + return err +} + +func (c *Client) ListRoleBindings(projectName string) ([]*RoleBinding, error) { + url := fmt.Sprintf("/rolebindings?projectName=%s", projectName) + + res := make([]*RoleBinding, 0) + _, err := c.Get(url, httpclient.SetResult(&res)) + + return res, err +} + +func (c *Client) CreateOrUpdateRoleBinding(projectName string, roleBinding *RoleBinding) error { + url := fmt.Sprintf("/rolebindings/%s?projectName=%s", roleBinding.Name, projectName) + _, err := c.Put(url, httpclient.SetBody(roleBinding)) + return err +} + +func (c *Client) CreateOrUpdateSystemRoleBinding(roleBinding *RoleBinding) error { + url := fmt.Sprintf("/system-rolebindings/%s", roleBinding.Name) + _, err := c.Put(url, httpclient.SetBody(roleBinding)) + return err +} + +func (c *Client) DeleteRoleBinding(name string, projectName string) error { + url := fmt.Sprintf("/rolebindings/%s?projectName=%s", name, projectName) + _, err := c.Delete(url) + return err +} + +type NameArgs struct { + Names []string `json:"names"` +} + +func (c *Client) DeleteRoleBindings(names []string, projectName string) error { + url := fmt.Sprintf("/rolebindings/bulk-delete?projectName=%s", projectName) + nameArgs := &NameArgs{} + for _, v := range names { + nameArgs.Names = append(nameArgs.Names, v) + } + _, err := c.Post(url, httpclient.SetBody(nameArgs)) + return err +} + +func (c *Client) DeleteRoles(names []string, projectName string) error { + url := fmt.Sprintf("/roles/bulk-delete?projectName=%s", projectName) + nameArgs := &NameArgs{} + for _, v := range names { + nameArgs.Names = append(nameArgs.Names, v) + } + _, err := c.Post(url, httpclient.SetBody(nameArgs)) + return err +} + +func (c *Client) CreateSystemRole(name string, role *Role) error { + url := fmt.Sprintf("/system-roles/%s", name) + _, err := c.Put(url, httpclient.SetBody(role)) + return err +} + +func (c *Client) CreatePresetRole(name string, role *Role) error { + url := fmt.Sprintf("/preset-roles/%s", name) + _, err := c.Put(url, httpclient.SetBody(role)) + return err +} + +type Role struct { + Name string `json:"name"` + Type setting.ResourceType `json:"type"` + Desc string `json:"desc"` + Rules []*struct { + Verbs []string `json:"verbs"` + Resources []string `json:"resources"` + Kind string `json:"kind"` + MatchAttributes []types.MatchAttribute `json:"match_attributes,omitempty"` + } `json:"rules"` +} + +func (c *Client) Healthz() error { + url := "/healthz" + _, err := c.Get(url) + return err +} diff --git a/pkg/shared/client/systemconfig/client.go b/pkg/shared/client/systemconfig/client.go new file mode 100644 index 0000000000000000000000000000000000000000..cc9052c52bd4843ed7fc010cbc520bda346e39aa --- /dev/null +++ b/pkg/shared/client/systemconfig/client.go @@ -0,0 +1,40 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package systemconfig + +import ( + "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +type Client struct { + *httpclient.Client + + host string +} + +func New() *Client { + host := config.ConfigServiceAddress() + c := httpclient.New( + httpclient.SetHostURL(host + "/api/v1"), + ) + + return &Client{ + Client: c, + host: host, + } +} diff --git a/pkg/shared/client/systemconfig/codehost.go b/pkg/shared/client/systemconfig/codehost.go new file mode 100644 index 0000000000000000000000000000000000000000..c4a557d0478a128c36f3abb56e8f88263e132301 --- /dev/null +++ b/pkg/shared/client/systemconfig/codehost.go @@ -0,0 +1,133 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package systemconfig + +import ( + "fmt" + + "github.com/koderover/zadig/pkg/tool/httpclient" + "github.com/koderover/zadig/pkg/types" +) + +const ( + GitLabProvider = "gitlab" + GitHubProvider = "github" + GerritProvider = "gerrit" + CodeHubProvider = "codehub" + GiteeProvider = "gitee" + OtherProvider = "other" +) + +type CodeHost struct { + ID int `json:"id"` + Address string `json:"address"` + Type string `json:"type"` + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + Namespace string `json:"namespace"` + Region string `json:"region"` + // the field and tag not consistent because of db field + AccessKey string `json:"application_id"` + SecretKey string `json:"client_secret"` + Username string `json:"username"` + Password string `json:"password"` + // the field determine whether the proxy is enabled + EnableProxy bool `json:"enable_proxy"` + UpdatedAt int64 `json:"updated_at"` + Alias string `json:"alias,omitempty"` + AuthType types.AuthType `json:"auth_type,omitempty"` + SSHKey string `json:"ssh_key,omitempty"` + PrivateAccessToken string `json:"private_access_token,omitempty"` +} + +type Option struct { + CodeHostType string + Address string + Namespace string +} + +func GetCodeHostInfo(opt *Option) (*CodeHost, error) { + return New().GetCodeHostByAddressAndOwner(opt.Address, opt.Namespace, opt.CodeHostType) +} + +func (c *Client) GetCodeHost(id int) (*CodeHost, error) { + url := fmt.Sprintf("/codehosts/%d", id) + + res := &CodeHost{} + _, err := c.Get(url, httpclient.SetResult(res)) + if err != nil { + return nil, err + } + + return res, nil +} + +func (c *Client) GetRawCodeHost(id int) (*CodeHost, error) { + url := fmt.Sprintf("/codehosts/%d?ignoreDelete=true", id) + + res := &CodeHost{} + _, err := c.Get(url, httpclient.SetResult(res)) + if err != nil { + return nil, err + } + + return res, nil +} + +func (c *Client) ListCodeHostsInternal() ([]*CodeHost, error) { + url := "/codehosts/internal" + + res := make([]*CodeHost, 0) + _, err := c.Get(url, httpclient.SetResult(&res)) + if err != nil { + return nil, err + } + + return res, nil +} + +func (c *Client) UpdateCodeHost(id int, codehost *CodeHost) error { + url := fmt.Sprintf("/codehosts/%d", id) + + _, err := c.Patch(url, httpclient.SetBody(codehost)) + return err +} + +func (c *Client) GetCodeHostByAddressAndOwner(address, owner, source string) (*CodeHost, error) { + url := "/codehosts/internal" + + res := make([]*CodeHost, 0) + + req := map[string]string{ + "address": address, + "owner": owner, + "source": source, + } + + _, err := c.Get(url, httpclient.SetQueryParams(req), httpclient.SetResult(&res)) + if err != nil { + return nil, err + } + + if len(res) == 0 { + return nil, fmt.Errorf("no codehost found") + } else if len(res) > 1 { + return nil, fmt.Errorf("more than one codehosts found") + } + + return res[0], nil +} diff --git a/pkg/shared/poetry/team.go b/pkg/shared/client/systemconfig/connector.go similarity index 44% rename from pkg/shared/poetry/team.go rename to pkg/shared/client/systemconfig/connector.go index 4b7d096d3501ebf3daa99947ce36df2fa036f2d9..13bd369dc5b38c11640646f2f4bcb5e4e6cee065 100644 --- a/pkg/shared/poetry/team.go +++ b/pkg/shared/client/systemconfig/connector.go @@ -14,48 +14,55 @@ See the License for the specific language governing permissions and limitations under the License. */ -package poetry +package systemconfig import ( - "fmt" + "encoding/json" - "go.uber.org/zap" + "github.com/dexidp/dex/connector/ldap" "github.com/koderover/zadig/pkg/tool/httpclient" ) -type Team struct { - ID int `json:"id"` - OrgID int `json:"orgId"` - Name string `json:"name"` - Desc string `json:"desc"` - IsTeamLeader bool `json:"isTeamLeader"` - Users []*UserInfo `json:"leaders"` - CreatedAt int64 `json:"created_at"` - UpdatedAt int64 `json:"updated_at"` +type Connector struct { + Type string `json:"type"` + ID string `json:"id"` + Name string `json:"name"` + Config interface{} `json:"config"` } -func (c *Client) ListTeams(orgID int, log *zap.SugaredLogger) ([]*Team, error) { - url := "/directory/teamss/search" - resp := make([]*Team, 0) - _, err := c.Get(url, httpclient.SetQueryParam("orgId", fmt.Sprintf("%d", orgID)), httpclient.SetResult(&resp)) +func (c *Client) GetLDAPConnector(id string) (*Connector, error) { + url := "/connectors/" + id + res := &Connector{} + _, err := c.Get(url, httpclient.SetResult(res)) if err != nil { - log.Errorf("ListTeams error: %v", err) return nil, err } - return resp, nil + configData, err := json.Marshal(res.Config) + if err != nil { + return nil, err + } + + ldapConfig := &ldap.Config{} + if err = json.Unmarshal(configData, ldapConfig); err != nil { + return nil, err + } + + res.Config = ldapConfig + + return res, err } -func (c *Client) GetTeam(teamID int) (*Team, error) { - url := fmt.Sprintf("/directory/teams/%d", teamID) - resp := new(Team) - _, err := c.Get(url, httpclient.SetResult(&resp)) +func (c *Client) ListConnectorsInternal() ([]*Connector, error) { + url := "/connectors/internal" + res := make([]*Connector, 0) + _, err := c.Get(url, httpclient.SetResult(&res)) if err != nil { return nil, err } - return resp, nil + return res, err } diff --git a/pkg/shared/poetry/org.go b/pkg/shared/client/systemconfig/email.go similarity index 60% rename from pkg/shared/poetry/org.go rename to pkg/shared/client/systemconfig/email.go index 463a7c07e803ba70b16a26daa97636fd0301c4f8..3e94ffc9d63399ca1c9914b4e8e00b1a787d0f82 100644 --- a/pkg/shared/poetry/org.go +++ b/pkg/shared/client/systemconfig/email.go @@ -14,33 +14,27 @@ See the License for the specific language governing permissions and limitations under the License. */ -package poetry +package systemconfig import ( - "fmt" - "github.com/koderover/zadig/pkg/tool/httpclient" ) -const ( - DefaultOrganization = iota + 1 -) - -type Organization struct { - ID int `json:"id"` - Name string `json:"name"` - Token string `json:"orgToken"` - Website string `json:"website"` +type Email struct { + Name string `json:"name"` + Port int `json:"port"` + UserName string `json:"username"` + Password string `json:"password"` } -func (c *Client) GetOrganization(orgID int) (*Organization, error) { - url := fmt.Sprintf("/directory/organization/%d", orgID) +func (c *Client) GetEmailHost() (*Email, error) { + url := "/emails/host/internal/" - org := &Organization{} - _, err := c.Get(url, httpclient.SetResult(org)) + res := &Email{} + _, err := c.Get(url, httpclient.SetResult(res)) if err != nil { return nil, err } - return org, nil + return res, err } diff --git a/pkg/shared/poetry/jira.go b/pkg/shared/client/systemconfig/jira.go similarity index 78% rename from pkg/shared/poetry/jira.go rename to pkg/shared/client/systemconfig/jira.go index 88fd8d7233c4f321a013d87e33270e9e11f37302..806d6204aeac4557f7e2ec7d4c8f45c5ecc092d7 100644 --- a/pkg/shared/poetry/jira.go +++ b/pkg/shared/client/systemconfig/jira.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package poetry +package systemconfig import ( "github.com/koderover/zadig/pkg/tool/httpclient" @@ -24,25 +24,20 @@ type JiraInfo struct { ID int64 `json:"id"` Host string `json:"host"` User string `json:"user"` - AccessToken string `json:"accessToken"` + AccessToken string `json:"access_token"` OrganizationID int `json:"organizationId"` CreatedAt int64 `json:"created_at"` UpdatedAt int64 `json:"updated_at"` } func (c *Client) GetJiraInfo() (*JiraInfo, error) { - url := "/directory/jira" + url := "/jira/internal" jira := &JiraInfo{} - _, err := c.Get(url, httpclient.SetResult(jira), httpclient.SetQueryParam("orgId", "1")) + _, err := c.Get(url, httpclient.SetResult(jira)) if err != nil { return nil, err } return jira, nil } - -func GetJiraInfo(host, token string) (*JiraInfo, error) { - c := New(host, token) - return c.GetJiraInfo() -} diff --git a/pkg/shared/client/user/client.go b/pkg/shared/client/user/client.go new file mode 100644 index 0000000000000000000000000000000000000000..5e6d8ac15e89bad3c8d9917c9d714db7b45001be --- /dev/null +++ b/pkg/shared/client/user/client.go @@ -0,0 +1,41 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package user + +import ( + "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +type Client struct { + *httpclient.Client + + host string +} + +func New() *Client { + host := config.UserServiceAddress() + + c := httpclient.New( + httpclient.SetHostURL(host + "/api/v1"), + ) + + return &Client{ + Client: c, + host: host, + } +} diff --git a/pkg/shared/client/user/user.go b/pkg/shared/client/user/user.go new file mode 100644 index 0000000000000000000000000000000000000000..d420163f67f26b2208c13a7140c4c9a2467f1627 --- /dev/null +++ b/pkg/shared/client/user/user.go @@ -0,0 +1,93 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package user + +import ( + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +type User struct { + UID string `json:"uid"` + Name string `json:"name"` + Email string `json:"email"` + Phone string `json:"phone"` + IdentityType string `json:"identity_type"` + Account string `json:"account"` +} + +type usersResp struct { + Users []*User `json:"users"` +} + +type SearchArgs struct { + UIDs []string `json:"uids"` +} + +func (c *Client) ListUsers(args *SearchArgs) ([]*User, error) { + url := "/users/search" + + res := &usersResp{} + _, err := c.Post(url, httpclient.SetResult(res), httpclient.SetBody(args)) + if err != nil { + return nil, err + } + + return res.Users, err +} + +type CreateUserArgs struct { + Name string `json:"name"` + Password string `json:"password"` + Email string `json:"email"` + Phone string `json:"phone"` + Account string `json:"account"` +} + +type CreateUserResp struct { + Name string `json:"name"` + Account string `json:"account"` + Uid string `json:"uid"` +} + +func (c *Client) CreateUser(args *CreateUserArgs) (*CreateUserResp, error) { + url := "/users" + resp := &CreateUserResp{} + _, err := c.Post(url, httpclient.SetBody(args), httpclient.SetResult(resp)) + return resp, err +} + +type SearchUserArgs struct { + Account string `json:"account"` +} + +type SearchUserResp struct { + TotalCount int `json:"totalCount"` + Users []*User `json:"users"` +} + +func (c *Client) SearchUser(args *SearchUserArgs) (*SearchUserResp, error) { + url := "/users/search" + resp := &SearchUserResp{} + _, err := c.Post(url, httpclient.SetBody(args), httpclient.SetResult(resp)) + return resp, err +} + +func (c *Client) Healthz() error { + url := "/healthz" + _, err := c.Get(url) + return err +} diff --git a/pkg/shared/codehost/codehost.go b/pkg/shared/codehost/codehost.go deleted file mode 100644 index 67b8d5f75e98e9c5d95ae81d27f59c61702afa57..0000000000000000000000000000000000000000 --- a/pkg/shared/codehost/codehost.go +++ /dev/null @@ -1,148 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package codehost - -import ( - "errors" - "strings" - - "github.com/koderover/zadig/pkg/config" - "github.com/koderover/zadig/pkg/shared/poetry" -) - -const ( - GitLabProvider = "gitlab" - GitHubProvider = "github" - GerritProvider = "gerrit" - CodeHubProvider = "codehub" - IlyshinProvider = "ilyshin" -) - -type CodeHost struct { - ID int `json:"id"` - OrgID int `json:"orgId"` - Address string `json:"address"` - Type string `json:"type"` - AccessToken string `json:"accessToken"` - Namespace string `json:"namespace"` -} - -type Option struct { - CodeHostType string - Address string - Namespace string - CodeHostID int -} - -type Detail struct { - ID int `json:"id"` - Name string `json:"name"` - Address string `json:"address"` - Owner string `json:"repoowner"` - Source string `json:"source"` - OauthToken string `json:"oauth_token"` - Region string `json:"region"` - Username string `json:"username"` - Password string `json:"password"` - AccessKey string `json:"applicationId"` - SecretKey string `json:"clientSecret"` -} - -func GetCodeHostList() ([]*poetry.CodeHost, error) { - poetryClient := poetry.New(config.PoetryServiceAddress(), config.PoetryAPIRootKey()) - return poetryClient.ListCodeHosts() -} - -func GetCodeHostInfo(option *Option) (*poetry.CodeHost, error) { - codeHosts, err := GetCodeHostList() - if err != nil { - return nil, err - } - - for _, codeHost := range codeHosts { - if option.CodeHostID != 0 && codeHost.ID == option.CodeHostID { - return codeHost, nil - } else if option.CodeHostID == 0 && option.CodeHostType != "" { - switch option.CodeHostType { - case GitHubProvider: - ns := strings.ToLower(codeHost.Namespace) - if strings.Contains(option.Address, codeHost.Address) && option.Namespace == ns { - return codeHost, nil - } - default: - if strings.Contains(option.Address, codeHost.Address) { - return codeHost, nil - } - } - } - } - - return nil, errors.New("not find codeHost") -} - -func GetCodeHostInfoByID(id int) (*poetry.CodeHost, error) { - return GetCodeHostInfo(&Option{CodeHostID: id}) -} - -func GetCodehostDetail(codehostID int) (*Detail, error) { - codehost, err := GetCodeHostInfo(&Option{CodeHostID: codehostID}) - if err != nil { - return nil, err - } - detail := &Detail{ - codehostID, - "", - codehost.Address, - codehost.Namespace, - codehost.Type, - codehost.AccessToken, - codehost.Region, - codehost.Username, - codehost.Password, - codehost.AccessKey, - codehost.SecretKey, - } - - return detail, nil -} - -func ListCodehostDetial() ([]*Detail, error) { - codehosts, err := GetCodeHostList() - if err != nil { - return nil, err - } - - var details []*Detail - - for _, codehost := range codehosts { - details = append(details, &Detail{ - codehost.ID, - "", - codehost.Address, - codehost.Namespace, - codehost.Type, - codehost.AccessToken, - codehost.Region, - codehost.Username, - codehost.Password, - codehost.AccessKey, - codehost.SecretKey, - }) - } - - return details, nil -} diff --git a/pkg/microservice/aslan/core/common/service/ilyshin/client.go b/pkg/shared/config/client.go similarity index 75% rename from pkg/microservice/aslan/core/common/service/ilyshin/client.go rename to pkg/shared/config/client.go index d2fc77361825faa6bbb2e4cd9cb7b3c53eb2c3c2..cc072ffa4a2c6d8e403c57220a89381030cc17ba 100644 --- a/pkg/microservice/aslan/core/common/service/ilyshin/client.go +++ b/pkg/shared/config/client.go @@ -14,19 +14,25 @@ See the License for the specific language governing permissions and limitations under the License. */ -package ilyshin +package config import ( - "github.com/koderover/zadig/pkg/tool/ilyshin" + "github.com/koderover/zadig/pkg/tool/httpclient" ) type Client struct { - *ilyshin.Client + *httpclient.Client + + host string } -func NewClient(address, accessToken string) *Client { - c := ilyshin.NewClient(address, accessToken) +func New(host string) *Client { + c := httpclient.New( + httpclient.SetHostURL(host), + ) + return &Client{ Client: c, + host: host, } } diff --git a/pkg/shared/poetry/check.go b/pkg/shared/config/features.go similarity index 70% rename from pkg/shared/poetry/check.go rename to pkg/shared/config/features.go index d205c5121359b8ef13f4e891f6b645eb82ca084c..38b78319ec3a6fab7eebec9a0c1a2a20c2529769 100644 --- a/pkg/shared/poetry/check.go +++ b/pkg/shared/config/features.go @@ -14,24 +14,27 @@ See the License for the specific language governing permissions and limitations under the License. */ -package poetry +package config import ( + "fmt" + "github.com/koderover/zadig/pkg/tool/httpclient" ) -type features struct { - Features []string `json:"features"` +type feature struct { + Name string `json:"name"` + Enabled bool `json:"enabled"` } -func (c *Client) ListFeatures() ([]string, error) { - url := "/directory/check" +func (c *Client) CheckFeature(featureName string) (bool, error) { + url := fmt.Sprintf("/api/v1/features/%s", featureName) - fs := &features{} + fs := &feature{} _, err := c.Get(url, httpclient.SetResult(fs)) if err != nil { - return nil, err + return false, err } - return fs.Features, nil + return fs.Enabled, nil } diff --git a/pkg/shared/handler/base.go b/pkg/shared/handler/base.go index c59719d52a726f6df1d9bda42da2fbcbc7255f91..e1727c72a076383e488aef8e2bfb8700d985dc52 100644 --- a/pkg/shared/handler/base.go +++ b/pkg/shared/handler/base.go @@ -17,50 +17,112 @@ limitations under the License. package handler import ( + "encoding/base64" + "encoding/json" + "fmt" "reflect" + "strings" + "time" "github.com/gin-gonic/gin" + "github.com/golang-jwt/jwt" "go.uber.org/zap" - "github.com/koderover/zadig/pkg/config" + systemmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/system/repository/models" + systemservice "github.com/koderover/zadig/pkg/microservice/aslan/core/system/service" "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/shared/client/aslan" - "github.com/koderover/zadig/pkg/shared/poetry" - "github.com/koderover/zadig/pkg/types/permission" "github.com/koderover/zadig/pkg/util/ginzap" ) // Context struct type Context struct { - Logger *zap.SugaredLogger - Err error - Resp interface{} - Username string - User *permission.User - RequestID string + Logger *zap.SugaredLogger + Err error + Resp interface{} + Account string + UserName string + UserID string + IdentityType string + RequestID string } +type jwtClaims struct { + Name string `json:"name"` + Email string `json:"email"` + UID string `json:"uid"` + Account string `json:"preferred_username"` + FederatedClaims FederatedClaims `json:"federated_claims"` + jwt.StandardClaims +} + +type FederatedClaims struct { + ConnectorId string `json:"connector_id"` + UserId string `json:"user_id"` +} + +// TODO: We need to implement a `context.Context` that conforms to the golang standard library. func NewContext(c *gin.Context) *Context { + logger := ginzap.WithContext(c).Sugar() + var claims jwtClaims + + token := c.GetHeader(setting.AuthorizationHeader) + if len(token) > 0 { + var err error + claims, err = getUserFromJWT(token) + if err != nil { + logger.Warnf("Failed to get user from token, err: %s", err) + } + } else { + claims.Name = "system" + } + return &Context{ - Username: currentUsername(c), - User: currentUser(c), - Logger: ginzap.WithContext(c).Sugar(), - RequestID: c.GetString(setting.RequestID), + UserName: claims.Name, + UserID: claims.UID, + Account: claims.Account, + IdentityType: claims.FederatedClaims.ConnectorId, + Logger: ginzap.WithContext(c).Sugar(), + RequestID: c.GetString(setting.RequestID), } } -// CurrentUser return current session user -func currentUser(c *gin.Context) *permission.User { - userInfo, isExist := c.Get(setting.SessionUser) - user, ok := userInfo.(*poetry.UserInfo) - if isExist && ok { - return poetry.ConvertUserInfo(user) +func GetResourcesInHeader(c *gin.Context) ([]string, bool) { + _, ok := c.Request.Header[setting.ResourcesHeader] + if !ok { + return nil, false + } + + res := c.GetHeader(setting.ResourcesHeader) + if res == "" { + return nil, true + } + var resources []string + if err := json.Unmarshal([]byte(res), &resources); err != nil { + return nil, false } - return permission.AnonymousUser + + return resources, true } -func currentUsername(c *gin.Context) (username string) { - return c.GetString(setting.SessionUsername) +func getUserFromJWT(token string) (jwtClaims, error) { + cs := jwtClaims{} + + parts := strings.Split(token, ".") + if len(parts) != 3 { + return cs, fmt.Errorf("compact JWS format must have three parts") + } + + payload, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil { + return cs, err + } + + err = json.Unmarshal(payload, &cs) + if err != nil { + return cs, err + } + + return cs, nil } func JSONResponse(c *gin.Context, ctx *Context) { @@ -78,11 +140,21 @@ func JSONResponse(c *gin.Context, ctx *Context) { // InsertOperationLog 插入操作日志 func InsertOperationLog(c *gin.Context, username, productName, method, function, detail, requestBody string, logger *zap.SugaredLogger) { - operationLogID, err := aslan.New(config.AslanServiceAddress(), config.PoetryAPIRootKey()).AddAuditLog(username, productName, method, function, detail, requestBody, logger) + req := &systemmodels.OperationLog{ + Username: username, + ProductName: productName, + Method: method, + Function: function, + Name: detail, + RequestBody: requestBody, + Status: 0, + CreatedAt: time.Now().Unix(), + } + operationLogID, err := systemservice.InsertOperation(req, logger) if err != nil { logger.Errorf("InsertOperation err:%v", err) } - c.Set("operationLogID", operationLogID) + c.Set("operationLogID", operationLogID.OperationLogID) } // responseHelper recursively finds all nil slice in the given interface, diff --git a/pkg/shared/kube/client/client.go b/pkg/shared/kube/client/client.go new file mode 100644 index 0000000000000000000000000000000000000000..c7581c4b7d30852d9862e24b25d753bb6e082568 --- /dev/null +++ b/pkg/shared/kube/client/client.go @@ -0,0 +1,73 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/kube/multicluster" +) + +func GetKubeClient(hubServerAddr, clusterID string) (client.Client, error) { + if clusterID == setting.LocalClusterID { + clusterID = "" + } + + return multicluster.GetKubeClient(hubServerAddr, clusterID) +} + +func GetKubeClientSet(hubServerAddr, clusterID string) (*kubernetes.Clientset, error) { + if clusterID == setting.LocalClusterID { + clusterID = "" + } + + return multicluster.GetKubeClientSet(hubServerAddr, clusterID) +} + +func GetDynamicKubeClient(hubserverAddr, clusterID string) (dynamic.Interface, error) { + if clusterID == setting.LocalClusterID { + clusterID = "" + } + + return multicluster.GetDynamicKubeclient(hubserverAddr, clusterID) +} + +func GetKubeAPIReader(hubServerAddr, clusterID string) (client.Reader, error) { + if clusterID == setting.LocalClusterID { + clusterID = "" + } + return multicluster.GetKubeAPIReader(hubServerAddr, clusterID) +} + +func GetRESTConfig(hubServerAddr, clusterID string) (*rest.Config, error) { + if clusterID == setting.LocalClusterID { + clusterID = "" + } + return multicluster.GetRESTConfig(hubServerAddr, clusterID) +} + +func GetClientset(hubServerAddr, clusterID string) (kubernetes.Interface, error) { + if clusterID == setting.LocalClusterID { + clusterID = "" + } + + return multicluster.GetClientset(hubServerAddr, clusterID) +} diff --git a/pkg/shared/kube/client/version.go b/pkg/shared/kube/client/version.go new file mode 100644 index 0000000000000000000000000000000000000000..e4a5639a984fafb6ab24f1e4372910a723ee9752 --- /dev/null +++ b/pkg/shared/kube/client/version.go @@ -0,0 +1,38 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "k8s.io/apimachinery/pkg/util/version" + k8sversion "k8s.io/apimachinery/pkg/version" +) + +var KubernetesVersion122 *version.Version + +func init() { + // as of zadig v1.11.0. Only kubernetes version 1.17+ is supported + // There is only 1 major api version change from v1.17+ + // More versions of kubernetes should be added if there are more API changes + // in future kubernetes version + v122, _ := version.ParseGeneric("v1.22.0") + KubernetesVersion122 = v122 +} + +func VersionLessThan122(ver *k8sversion.Info) bool { + currVersion, _ := version.ParseGeneric(ver.String()) + return currVersion.LessThan(KubernetesVersion122) +} diff --git a/pkg/shared/kube/resource/namespace.go b/pkg/shared/kube/resource/namespace.go index 7fad232c187ab457fff39ed8448543ffb6e180ec..50fc6e1bea7806aaad84c34a856e5793211fdff5 100644 --- a/pkg/shared/kube/resource/namespace.go +++ b/pkg/shared/kube/resource/namespace.go @@ -21,4 +21,6 @@ type Namespace struct { Status string `json:"status"` Age string `json:"age"` Labels map[string]string `json:"labels"` + // Whether it is the namespace currently installed by zadig + Current bool `json:"current"` } diff --git a/pkg/shared/kube/resource/node.go b/pkg/shared/kube/resource/node.go new file mode 100644 index 0000000000000000000000000000000000000000..733ef420426875e31d72dcc799853a565531a26d --- /dev/null +++ b/pkg/shared/kube/resource/node.go @@ -0,0 +1,23 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resource + +type Node struct { + Labels []string `json:"labels"` + Ready bool `json:"ready"` + IP string `json:"ip"` +} diff --git a/pkg/shared/kube/resource/pod.go b/pkg/shared/kube/resource/pod.go index d74d6537db917e87b823d275f480b9e93c93aef9..2c1c04f80da3703662db0c3e6b418eabf2c31baf 100644 --- a/pkg/shared/kube/resource/pod.go +++ b/pkg/shared/kube/resource/pod.go @@ -17,14 +17,20 @@ limitations under the License. package resource type Pod struct { - Kind string `json:"kind"` - Name string `json:"name"` - Status string `json:"status"` - Age string `json:"age"` - CreateTime int64 `json:"createtime"` - IP string `json:"ip"` - Labels map[string]string `json:"labels"` - ContainerStatuses []Container `json:"containers"` + Kind string `json:"kind"` + Name string `json:"name"` + Status string `json:"status"` + Age string `json:"age"` + CreateTime int64 `json:"createtime"` + IP string `json:"ip"` + Labels map[string]string `json:"labels"` + ContainerStatuses []Container `json:"containers"` + NodeName string `json:"node_name"` + HostIP string `json:"host_ip"` + EnableDebugContainer bool `json:"enable_debug_container"` + PodReady bool `json:"pod_ready"` + ContainersReady bool `json:"containers_ready"` + ContainersMessage string `json:"containers_message"` } type Container struct { @@ -32,6 +38,7 @@ type Container struct { Image string `json:"image"` RestartCount int32 `json:"restart_count"` Status string `json:"status"` + Ready bool `json:"ready"` // Message regarding the last termination of the container Message string `json:"message"` // reason from the last termination of the container diff --git a/pkg/shared/kube/resource/workload.go b/pkg/shared/kube/resource/workload.go index fe8be9fc6d89fe6df857041209cf07882e762956..fa7131686c263bd80ffcb56859e7c046f37420d4 100644 --- a/pkg/shared/kube/resource/workload.go +++ b/pkg/shared/kube/resource/workload.go @@ -25,6 +25,7 @@ type Workload struct { } type ContainerImage struct { - Name string `json:"name"` - Image string `json:"image"` + Name string `json:"name"` + Image string `json:"image"` + ImageName string `json:"image_name"` } diff --git a/pkg/shared/kube/wrapper/deployment.go b/pkg/shared/kube/wrapper/deployment.go index 5cd417f7a39c2bf9446b2b7b1c0337f5fae918b5..5edfacf44c6c8b7dfebd3f30e4d53ba52b9f66a7 100644 --- a/pkg/shared/kube/wrapper/deployment.go +++ b/pkg/shared/kube/wrapper/deployment.go @@ -22,6 +22,7 @@ import ( "github.com/koderover/zadig/pkg/setting" "github.com/koderover/zadig/pkg/shared/kube/resource" + "github.com/koderover/zadig/pkg/util" ) // deployment is the wrapper for appsv1.Deployment type. @@ -56,8 +57,9 @@ func (w *deployment) WorkloadResource(pods []*corev1.Pod) *resource.Workload { Pods: make([]*resource.Pod, 0, len(pods)), } + // Note: EphemeralContainers only belong to Pods and do not exist in Deployment Spec. for _, c := range w.Spec.Template.Spec.Containers { - wl.Images = append(wl.Images, resource.ContainerImage{Name: c.Name, Image: c.Image}) + wl.Images = append(wl.Images, resource.ContainerImage{Name: c.Name, Image: c.Image, ImageName: util.ExtractImageName(c.Image)}) } for _, p := range pods { @@ -80,3 +82,15 @@ func (w *deployment) ImageInfos() (images []string) { } return } + +func (w *deployment) GetKind() string { + return w.Kind +} + +func (w *deployment) GetContainers() []*resource.ContainerImage { + containers := make([]*resource.ContainerImage, 0, len(w.Spec.Template.Spec.Containers)) + for _, c := range w.Spec.Template.Spec.Containers { + containers = append(containers, &resource.ContainerImage{Name: c.Name, Image: c.Image}) + } + return containers +} diff --git a/pkg/shared/kube/wrapper/event.go b/pkg/shared/kube/wrapper/event.go index a2b758f59c3c848cd04e48174e23a79b6e3279a4..1943e387759f787f929d2a4c60cb93dc3f7550ee 100644 --- a/pkg/shared/kube/wrapper/event.go +++ b/pkg/shared/kube/wrapper/event.go @@ -43,7 +43,7 @@ func (w *event) Unwrap() *corev1.Event { } func (w *event) Resource() *resource.Event { - return &resource.Event{ + e := &resource.Event{ Reason: w.Reason, Message: w.Message, FirstSeen: w.FirstTimestamp.Unix(), @@ -51,4 +51,9 @@ func (w *event) Resource() *resource.Event { Count: w.Count, Type: w.Type, } + if (w.FirstTimestamp.IsZero() || w.LastTimestamp.IsZero()) && !w.EventTime.IsZero() { + e.FirstSeen = w.EventTime.Unix() + e.LastSeen = w.EventTime.Unix() + } + return e } diff --git a/pkg/shared/kube/wrapper/ingress.go b/pkg/shared/kube/wrapper/ingress.go index e6e250a6f7cdebc1f5f861f6d6137945d9d40138..5b9bac2d77e7019e504fe13d25445a1abbf7ad06 100644 --- a/pkg/shared/kube/wrapper/ingress.go +++ b/pkg/shared/kube/wrapper/ingress.go @@ -18,6 +18,7 @@ package wrapper import ( extensionsv1beta1 "k8s.io/api/extensions/v1beta1" + v1 "k8s.io/api/networking/v1" "github.com/koderover/zadig/pkg/shared/kube/resource" "github.com/koderover/zadig/pkg/util" @@ -43,6 +44,32 @@ func (ing *ingress) Unwrap() *extensionsv1beta1.Ingress { return ing.Ingress } +func GetIngressHostInfo(ing *v1.Ingress) []resource.HostInfo { + var res []resource.HostInfo + for _, rule := range ing.Spec.Rules { + info := resource.HostInfo{ + Host: rule.Host, + Backends: []resource.Backend{}, + } + + if rule.HTTP == nil { + continue + } + + for _, path := range rule.HTTP.Paths { + backend := resource.Backend{ + ServiceName: path.Backend.Service.Name, + ServicePort: string(path.Backend.Service.Port.Number), + } + info.Backends = append(info.Backends, backend) + } + + res = append(res, info) + } + + return res +} + func (ing *ingress) HostInfo() []resource.HostInfo { var res []resource.HostInfo for _, rule := range ing.Spec.Rules { diff --git a/pkg/shared/kube/wrapper/namespace.go b/pkg/shared/kube/wrapper/namespace.go index 4faa5d727bcdcd310f7b56e37b14914c512fd027..3fd58ef1eb237c9d479cdda02c132ef84722f4e2 100644 --- a/pkg/shared/kube/wrapper/namespace.go +++ b/pkg/shared/kube/wrapper/namespace.go @@ -19,6 +19,7 @@ package wrapper import ( corev1 "k8s.io/api/core/v1" + "github.com/koderover/zadig/pkg/config" "github.com/koderover/zadig/pkg/shared/kube/resource" "github.com/koderover/zadig/pkg/util" ) @@ -53,9 +54,10 @@ func (w *namespace) Terminating() bool { func (w *namespace) Resource() *resource.Namespace { return &resource.Namespace{ - Name: w.Name, - Status: string(w.Status.Phase), - Age: util.Age(w.CreationTimestamp.Unix()), - Labels: w.Labels, + Name: w.Name, + Status: string(w.Status.Phase), + Age: util.Age(w.CreationTimestamp.Unix()), + Labels: w.Labels, + Current: w.Name == config.Namespace(), } } diff --git a/pkg/shared/kube/wrapper/pod.go b/pkg/shared/kube/wrapper/pod.go index 32902ec2a4b453a1a2193b5b309bfafd90fe2811..03b85f93bc5e92e13e41a3e261c21eff259e71d6 100644 --- a/pkg/shared/kube/wrapper/pod.go +++ b/pkg/shared/kube/wrapper/pod.go @@ -89,14 +89,19 @@ func (w *pod) Ready() bool { // All conditions specified in readinessGates are True. // When a Pod's containers are Ready but at least one custom condition is missing or False, the kubelet sets the Pod's condition to ContainersReady. // https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-readiness-status -func (w *pod) ContainersReady() bool { +func (w *pod) ContainersReady() (bool, string) { cs := w.Status.Conditions + for _, c := range cs { if c.Type == corev1.ContainersReady { - return c.Status == corev1.ConditionTrue + if c.Status == corev1.ConditionTrue { + return true, "" + } else { + return false, c.Message + } } } - return false + return false, "" } func (w *pod) Containers() []string { @@ -118,6 +123,7 @@ func (w *pod) ContainerNames() []string { } func (w *pod) Resource() *resource.Pod { + containersReady, containersMessage := w.ContainersReady() p := &resource.Pod{ Name: w.Name, Status: string(w.Status.Phase), @@ -125,16 +131,31 @@ func (w *pod) Resource() *resource.Pod { CreateTime: w.CreationTimestamp.Unix(), IP: w.Status.PodIP, Labels: w.Labels, + PodReady: w.Ready(), + ContainersReady: containersReady, + ContainersMessage: containersMessage, ContainerStatuses: []resource.Container{}, + NodeName: w.Spec.NodeName, + HostIP: w.Status.HostIP, } if len(w.OwnerReferences) > 0 { p.Kind = w.OwnerReferences[0].Kind } - for _, container := range w.Status.ContainerStatuses { + containersStatus := []corev1.ContainerStatus{} + containersStatus = append(containersStatus, w.Status.ContainerStatuses...) + if CheckEphemeralContainerStatusFieldExist(&w.Status) { + for _, status := range w.Status.EphemeralContainerStatuses { + status.Ready = true + containersStatus = append(containersStatus, status) + } + } + + for _, container := range containersStatus { cs := resource.Container{ Name: container.Name, RestartCount: container.RestartCount, + Ready: container.Ready, } if container.State.Running != nil { @@ -170,9 +191,34 @@ func (w *pod) Resource() *resource.Pod { break } } + + if CheckEphemeralContainerFieldExist(&w.Spec) { + for _, specContainer := range w.Spec.EphemeralContainers { + if specContainer.Name == container.Name { + cs.Image = specContainer.Image + break + } + } + } p.ContainerStatuses = append(p.ContainerStatuses, cs) } + // Note: Seems that in K8s versions [v1.16, v1.22], EphemeralContainerStatuses exist but are empty while in K8s versions + // [v1.23, ], EphemeralContainerStatuses exist and are not empty. + if CheckEphemeralContainerStatusFieldExist(&w.Status) && len(w.Status.EphemeralContainerStatuses) == 0 && + CheckEphemeralContainerFieldExist(&w.Spec) { + for _, container := range w.Spec.EphemeralContainers { + cs := resource.Container{ + Name: container.Name, + Image: container.Image, + RestartCount: 0, + Status: "running", + Ready: true, + } + p.ContainerStatuses = append(p.ContainerStatuses, cs) + } + } + if w.DeletionTimestamp != nil { p.Status = "Terminating" } @@ -186,5 +232,9 @@ func (w *pod) Resource() *resource.Pod { } } + if CheckEphemeralContainerFieldExist(&w.Spec) && len(w.Spec.EphemeralContainers) > 0 { + p.EnableDebugContainer = true + } + return p } diff --git a/pkg/shared/kube/wrapper/statefulset.go b/pkg/shared/kube/wrapper/statefulset.go index 0ae9c3332f0cdc3d59c6be9fa519a73233ab0643..6f36328200beede7b368a1ddc2b13cff23b56876 100644 --- a/pkg/shared/kube/wrapper/statefulset.go +++ b/pkg/shared/kube/wrapper/statefulset.go @@ -56,6 +56,7 @@ func (w *statefulSet) WorkloadResource(pods []*corev1.Pod) *resource.Workload { Pods: make([]*resource.Pod, 0, len(pods)), } + // Note: EphemeralContainers only belong to Pods and do not exist in Deployment Spec. for _, c := range w.Spec.Template.Spec.Containers { wl.Images = append(wl.Images, resource.ContainerImage{Name: c.Name, Image: c.Image}) } @@ -80,3 +81,15 @@ func (w *statefulSet) ImageInfos() (images []string) { } return } + +func (w *statefulSet) GetKind() string { + return w.Kind +} + +func (w *statefulSet) GetContainers() []*resource.ContainerImage { + containers := make([]*resource.ContainerImage, 0, len(w.Spec.Template.Spec.Containers)) + for _, c := range w.Spec.Template.Spec.Containers { + containers = append(containers, &resource.ContainerImage{Name: c.Name, Image: c.Image}) + } + return containers +} diff --git a/pkg/tool/kube/getter/serviceaccount.go b/pkg/shared/kube/wrapper/utils.go similarity index 55% rename from pkg/tool/kube/getter/serviceaccount.go rename to pkg/shared/kube/wrapper/utils.go index 825ed2460108c16421b2593e63390a46e0f56d33..2ce244ed3c8eef21df8b0e74cffc0bff43382dfb 100644 --- a/pkg/tool/kube/getter/serviceaccount.go +++ b/pkg/shared/kube/wrapper/utils.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The KodeRover Authors. +Copyright 2022 The KodeRover Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,19 +14,20 @@ See the License for the specific language governing permissions and limitations under the License. */ -package getter +package wrapper import ( + "reflect" + corev1 "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/client" ) -func GetServiceAccount(ns, name string, cl client.Client) (*corev1.ServiceAccount, bool, error) { - svc := &corev1.ServiceAccount{} - found, err := GetResourceInCache(ns, name, svc, cl) - if err != nil || !found { - svc = nil - } +func CheckEphemeralContainerFieldExist(podSpec *corev1.PodSpec) bool { + metaValue := reflect.ValueOf(podSpec).Elem() + return metaValue.FieldByName("EphemeralContainers") != (reflect.Value{}) +} - return svc, found, err +func CheckEphemeralContainerStatusFieldExist(podStatus *corev1.PodStatus) bool { + metaValue := reflect.ValueOf(podStatus).Elem() + return metaValue.FieldByName("EphemeralContainerStatuses") != (reflect.Value{}) } diff --git a/pkg/shared/poetry/actions.go b/pkg/shared/poetry/actions.go deleted file mode 100644 index bca860415b0435723089d637f65301be89d611c8..0000000000000000000000000000000000000000 --- a/pkg/shared/poetry/actions.go +++ /dev/null @@ -1,156 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package poetry - -import ( - "fmt" - "strconv" - - "go.uber.org/zap" - - e "github.com/koderover/zadig/pkg/tool/errors" - "github.com/koderover/zadig/pkg/tool/httpclient" -) - -func (c *Client) AddEnvRolePermission(productName, envName string, permissionUUIDs []string, roleID int, log *zap.SugaredLogger) (int, error) { - url := "/directory/roleEnv" - req := map[string]interface{}{ - "productName": productName, - "envName": envName, - "permissionUUIDs": permissionUUIDs, - "roleId": roleID, - } - - rm := &ResponseMessage{} - _, err := c.Post(url, httpclient.SetBody(req), httpclient.SetResult(rm)) - if err != nil { - log.Errorf("AddEnvRolePermission error: %v", err) - return 0, e.ErrListUsers.AddDesc(err.Error()) - } - - if rm.ResultCode == 0 { - return 1, nil - } - - return 0, e.ErrListUsers.AddDesc(fmt.Sprintf("ResultCode: %d", rm.ResultCode)) -} - -func (c *Client) DeleteEnvRolePermission(productName, envName string, log *zap.SugaredLogger) (int, error) { - url := "/directory/roleEnv" - qs := map[string]string{ - "productName": productName, - "envName": envName, - } - - rm := &ResponseMessage{} - _, err := c.Delete(url, httpclient.SetQueryParams(qs), httpclient.SetResult(rm)) - if err != nil { - log.Errorf("DeleteEnvRolePermission error: %v", err) - return 0, e.ErrListUsers.AddDesc(err.Error()) - } - - if rm.ResultCode == 0 { - return 1, nil - } - return 0, e.ErrListUsers.AddDesc(fmt.Sprintf("ResultCode: %d", rm.ResultCode)) -} - -func (c *Client) ListEnvRolePermission(productName, envName string, roleID int64, log *zap.SugaredLogger) ([]*EnvRolePermission, error) { - url := "/directory/roleEnv" - qs := map[string]string{ - "productName": productName, - "envName": envName, - "roleId": strconv.Itoa(int(roleID)), - } - - res := make([]*EnvRolePermission, 0) - _, err := c.Get(url, httpclient.SetQueryParams(qs), httpclient.SetResult(&res)) - if err != nil { - log.Errorf("GetEnvRolePermission error: %v", err) - return res, e.ErrListUsers.AddDesc(err.Error()) - } - - return res, nil -} - -func (c *Client) AddProductTeam(productName string, teamID int, userIDs []int, log *zap.SugaredLogger) (int, error) { - url := "/directory/teamProduct" - req := map[string]interface{}{ - "productName": productName, - "teamId": teamID, - "userIds": userIDs, - } - - rm := &ResponseMessage{} - _, err := c.Post(url, httpclient.SetBody(req), httpclient.SetResult(rm)) - if err != nil { - log.Errorf("AddProductTeam error: %v", err) - return 0, e.ErrCreateProductTeam.AddDesc(err.Error()) - } - - if rm.ResultCode == 0 { - return 1, nil - } - return 0, e.ErrCreateProductTeam.AddDesc(fmt.Sprintf("ResultCode: %d", rm.ResultCode)) -} - -func (c *Client) DeleteProductTeam(productName string, log *zap.SugaredLogger) error { - url := "/directory/teamProduct" - - rm := &ResponseMessage{} - _, err := c.Delete(url, httpclient.SetQueryParam("productName", productName), httpclient.SetResult(rm)) - if err != nil { - log.Errorf("DeleteProductTeam error: %v", err) - return e.ErrDeleteProductTeam.AddDesc(err.Error()) - } - if rm.ResultCode == 0 { - return nil - } - return e.ErrDeleteProductTeam.AddDesc(fmt.Sprintf("ResultCode: %d", rm.ResultCode)) -} - -func (c *Client) HasOperatePermission(productName, permissionUUID string, userID int, isSuperUser bool, log *zap.SugaredLogger) bool { - if isSuperUser { - return true - } - - res, err := c.GetUserPermissionUUIDMap(productName, permissionUUID, userID, log) - if err != nil { - return false - } - - return res["isContain"] -} - -func (c *Client) GetUserPermissionUUIDMap(productName, permissionUUID string, userID int, log *zap.SugaredLogger) (map[string]bool, error) { - url := "/directory/userPermissionRelation" - qs := map[string]string{ - "productName": productName, - "userId": strconv.Itoa(userID), - "permissionUUID": permissionUUID, - "permissionType": "2", - } - - res := make(map[string]bool) - _, err := c.Get(url, httpclient.SetQueryParams(qs), httpclient.SetResult(&res)) - if err != nil { - log.Errorf("GetUserPermissionUUIDMap error: %v", err) - return nil, err - } - - return res, nil -} diff --git a/pkg/shared/poetry/codehost.go b/pkg/shared/poetry/codehost.go deleted file mode 100644 index add22aa37316ce78c9dc8f5c815c5145a041f234..0000000000000000000000000000000000000000 --- a/pkg/shared/poetry/codehost.go +++ /dev/null @@ -1,47 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package poetry - -import ( - "github.com/koderover/zadig/pkg/tool/httpclient" -) - -type CodeHost struct { - ID int `json:"id"` - OrgID int `json:"orgId"` - Address string `json:"address"` - Type string `json:"type"` - AccessToken string `json:"accessToken"` - Namespace string `json:"namespace"` - Region string `json:"region"` - AccessKey string `json:"applicationId"` - SecretKey string `json:"clientSecret"` - Username string `json:"username"` - Password string `json:"password"` -} - -func (c *Client) ListCodeHosts() ([]*CodeHost, error) { - url := "/directory/codehostss/search" - - codeHosts := make([]*CodeHost, 0) - _, err := c.Get(url, httpclient.SetResult(&codeHosts), httpclient.SetQueryParam("orgId", "1")) - if err != nil { - return nil, err - } - - return codeHosts, nil -} diff --git a/pkg/shared/poetry/types.go b/pkg/shared/poetry/types.go deleted file mode 100644 index 3434f5004b282a06f1ec24d3fcc9153841fdad2d..0000000000000000000000000000000000000000 --- a/pkg/shared/poetry/types.go +++ /dev/null @@ -1,80 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package poetry - -import "github.com/koderover/zadig/pkg/types/permission" - -const ( - ProjectOwner = 3 -) - -type User interface { - GetName() string - GetEmail() string - GetPhone() string -} - -type UserInfo struct { - ID int `json:"id"` - Name string `json:"name"` - Email string `json:"email"` - Password string `json:"password"` - Phone string `json:"phone"` - IsAdmin bool `json:"isAdmin"` - IsSuperUser bool `json:"isSuperUser"` - IsTeamLeader bool `json:"isTeamLeader"` - OrganizationID int `json:"organization_id"` - Directory string `json:"directory"` - LastLoginAt int64 `json:"lastLogin"` - CreatedAt int64 `json:"created_at"` - UpdatedAt int64 `json:"updated_at"` -} - -type ResponseMessage struct { - ResultCode int `json:"resultCode"` - ErrorMsg string `json:"errorMsg"` -} - -type EnvRolePermission struct { - ID int64 `json:"id"` - ProductName string `json:"productName"` - EnvName string `json:"envName"` - RoleID int64 `json:"roleId"` - PermissionUUID string `json:"permissionUUID"` - RoleName string `json:"roleName"` -} - -func ConvertUserInfo(userInfo *UserInfo) *permission.User { - if userInfo == nil { - return nil - } - user := new(permission.User) - user.ID = userInfo.ID - user.Name = userInfo.Name - user.Email = userInfo.Email - user.Password = userInfo.Password - user.Phone = userInfo.Phone - user.IsAdmin = userInfo.IsAdmin - user.IsSuperUser = userInfo.IsSuperUser - user.IsTeamLeader = userInfo.IsTeamLeader - user.OrganizationID = userInfo.OrganizationID - user.Directory = userInfo.Directory - user.LastLoginAt = userInfo.LastLoginAt - user.CreatedAt = userInfo.CreatedAt - user.UpdatedAt = userInfo.UpdatedAt - return user -} diff --git a/pkg/shared/poetry/user.go b/pkg/shared/poetry/user.go deleted file mode 100644 index db5d535362e73c96387c288abb1346f9c1657584..0000000000000000000000000000000000000000 --- a/pkg/shared/poetry/user.go +++ /dev/null @@ -1,418 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package poetry - -import ( - "fmt" - "strconv" - "strings" - "time" - - "go.uber.org/zap" - - "github.com/koderover/zadig/pkg/setting" - "github.com/koderover/zadig/pkg/tool/httpclient" - "github.com/koderover/zadig/pkg/types/permission" -) - -//roleType -const ( - ProjectType = 2 - AllUsersRoleName = "all-users" -) - -type UserRole struct { - RoleID int64 `json:"roleId"` - UserID int `json:"userId"` - ProductName string `json:"productName"` -} - -type RoleProduct struct { - RoleID int64 `json:"roleId"` - ProductName string `json:"productName"` - Role Role `json:"role"` -} - -type UpdateUserRoleReq struct { - ProductName string `json:"productName"` - RoleID int64 `json:"roleId"` - RoleType int `json:"roleType"` - UserIDList []int `json:"userIds"` -} - -type RolePermission struct { - RoleID int64 `json:"roleId"` - PermissionUUID string `json:"permissionUUID"` -} - -type RoleEnv struct { - RoleID int `json:"roleId"` - EnvName string `json:"envName"` - ProductName string `json:"productName"` -} - -type RolePermissionReq struct { - RoleID int64 `json:"roleId"` - PermissionUUID []string `json:"permissionUUIDs"` - RoleType int `json:"roleType"` - ProductName string `json:"productName"` -} - -type Role struct { - ID int64 `json:"id"` - IsDisabled bool `json:"isDisabled"` - Name string `json:"name"` - ProductName string `json:"productName"` - RoleType int `json:"roleType"` - UpdateBy string `json:"updateBy"` - UpdateAt int64 `json:"updatedAt"` -} - -type UserEnv struct { - ProductName string `json:"productName"` - EnvName string `json:"envName"` - UserID int `json:"userId"` - PermissionUUID string `json:"permissionUUID"` -} - -func (c *Client) ListUserRoles(userID, roleType int, log *zap.SugaredLogger) ([]*UserRole, error) { - url := "/directory/roleUser" - - ur := make([]*UserRole, 0) - _, err := c.Get(url, httpclient.SetResult(&ur), httpclient.SetQueryParam("roleType", strconv.Itoa(roleType)), httpclient.SetQueryParam("userId", strconv.Itoa(userID))) - if err != nil { - log.Errorf("ListUserRoles error: %v", err) - return nil, err - } - - return ur, nil -} - -func (c *Client) ListPermissionUsers(productName string, roleID int64, roleType int, log *zap.SugaredLogger) ([]int, error) { - url := "/directory/roleUser" - qs := map[string]string{ - "productName": productName, - "roleType": strconv.Itoa(roleType), - "roleId": strconv.Itoa(int(roleID)), - } - - ur := make([]*UserRole, 0) - _, err := c.Get(url, httpclient.SetResult(&ur), httpclient.SetQueryParams(qs)) - if err != nil { - log.Errorf("ListPermissionUsers error: %v", err) - return nil, err - } - - var ret []int - for _, userRole := range ur { - ret = append(ret, userRole.UserID) - } - return ret, nil -} - -func (c *Client) UpdateUserRole(roleID int64, roleType int, productName string, userIDList []int, log *zap.SugaredLogger) error { - url := "/directory/roleUser" - req := &UpdateUserRoleReq{ - ProductName: productName, - RoleID: roleID, - RoleType: roleType, - UserIDList: userIDList, - } - - _, err := c.Post(url, httpclient.SetBody(req)) - if err != nil { - log.Errorf("UpdateUserRole failed, error: %v", err) - return err - } - - return nil -} - -func (c *Client) DeleteUserRole(roleID int64, roleType, userID int, productName string, log *zap.SugaredLogger) error { - url := "/directory/roleUser" - qs := map[string]string{ - "productName": productName, - "roleType": strconv.Itoa(roleType), - "roleId": strconv.Itoa(int(roleID)), - "userId": strconv.Itoa(userID), - } - - _, err := c.Delete(url, httpclient.SetQueryParams(qs)) - if err != nil { - log.Errorf("DeleteUserRole error: %v", err) - return err - } - return nil -} - -func (c *Client) ListRoleProjects(roleID int64, log *zap.SugaredLogger) ([]*RoleProduct, error) { - url := "/directory/roleProduct" - - rp := make([]*RoleProduct, 0) - _, err := c.Get(url, httpclient.SetResult(&rp), httpclient.SetQueryParam("roleId", strconv.Itoa(int(roleID)))) - if err != nil { - log.Errorf("ListRoleProjects error: %v", err) - return nil, err - } - - return rp, nil -} - -func (c *Client) ContributorRoleExist(productName string, log *zap.SugaredLogger) bool { - url := "/directory/roleProduct" - - rp := make([]*RoleProduct, 0) - _, err := c.Get(url, httpclient.SetResult(&rp), httpclient.SetQueryParam("productName", productName)) - if err != nil { - log.Errorf("Failed to list role project, error: %v", err) - return false - } - - for _, roleProduct := range rp { - if roleProduct.Role.Name == setting.RoleContributor { - return true - } - } - return false -} - -func (c *Client) GetContributorRoleID(productName string, log *zap.SugaredLogger) int64 { - url := "/directory/roleProduct" - - rp := make([]*RoleProduct, 0) - _, err := c.Get(url, httpclient.SetResult(&rp), httpclient.SetQueryParam("productName", productName)) - if err != nil { - log.Errorf("Failed to list role project, sendRequest error: %v", err) - return -1 - } - - for _, roleProduct := range rp { - if roleProduct.Role.Name == setting.RoleContributor { - return roleProduct.RoleID - } - } - return -1 -} - -func (c *Client) ListRolePermissions(roleID int64, roleType int, productName string, log *zap.SugaredLogger) ([]*RolePermission, error) { - url := "/directory/rolePermission" - qs := map[string]string{ - "productName": productName, - "roleType": strconv.Itoa(roleType), - "roleId": strconv.Itoa(int(roleID)), - } - - rp := make([]*RolePermission, 0) - _, err := c.Get(url, httpclient.SetResult(&rp), httpclient.SetQueryParams(qs)) - if err != nil { - log.Errorf("ListRolePermissions error: %v", err) - return nil, err - } - - return rp, nil -} - -func (c *Client) ListRoleEnvs(productName, envName string, roleID int64, log *zap.SugaredLogger) ([]*RoleEnv, error) { - url := "/directory/roleEnv" - qs := map[string]string{ - "productName": productName, - "envName": envName, - "roleId": strconv.Itoa(int(roleID)), - } - - re := make([]*RoleEnv, 0) - _, err := c.Get(url, httpclient.SetResult(&re), httpclient.SetQueryParams(qs)) - if err != nil { - log.Errorf("ListRoleEnvs error: %v", err) - return nil, err - } - - return re, nil -} - -func (c *Client) GetUserProject(userID int, log *zap.SugaredLogger) (map[string][]int64, error) { - productNameMap := make(map[string][]int64) - userRoles, err := c.ListUserRoles(userID, ProjectType, log) - if err != nil { - log.Errorf("GetUserProject ListUserRoles error: %v", err) - return productNameMap, fmt.Errorf("GetUserProject ListUserRoles error: %v", err) - } - for _, userRole := range userRoles { - productNameMap[userRole.ProductName] = append(productNameMap[userRole.ProductName], userRole.RoleID) - } - return productNameMap, nil -} - -func (c *Client) GetUserPermissionUUIDs(roleID int64, productName string, log *zap.SugaredLogger) ([]string, error) { - permissionUUIDs := make([]string, 0) - rolePermissions, err := c.ListRolePermissions(roleID, ProjectType, productName, log) - if err != nil { - log.Errorf("GetUserPermission ListRolePermissions error: %v", err) - return []string{}, fmt.Errorf("GetUserPermission ListRolePermissions error: %v", err) - } - for _, rolePermission := range rolePermissions { - permissionUUIDs = append(permissionUUIDs, rolePermission.PermissionUUID) - } - return permissionUUIDs, nil -} - -// ListProductPermissionUsers productName和permissionId为空时只返回所有管理员用户，不为空则返回所有管理员用户和有该权限的普通用户 -func (c *Client) ListProductPermissionUsers(productName, permissionID string, log *zap.SugaredLogger) ([]string, error) { - url := "/directory/productUser" - qs := map[string]string{ - "productName": productName, - "permissionId": permissionID, - } - - resp := make([]string, 0) - _, err := c.Get(url, httpclient.SetResult(&resp), httpclient.SetQueryParams(qs)) - if err != nil { - log.Errorf("ListProductPermissionUsers error: %v", err) - return nil, err - } - - return resp, nil -} - -func (c *Client) CreateContributorRole(product string, log *zap.SugaredLogger) error { - // create role first - url := "/directory/roles" - req := &Role{ - IsDisabled: false, - Name: setting.RoleContributor, - ProductName: product, - RoleType: ProjectType, - UpdateBy: setting.SystemUser, - UpdateAt: time.Now().Unix(), - } - - r := &Role{} - _, err := c.Post(url, httpclient.SetBody(req), httpclient.SetResult(r)) - if err != nil { - log.Errorf("CreateContributorRole failed, error: %v", err) - return err - } - - // 然后创建role product - roleProductURL := "/directory/roleProduct" - roleProductReq := RoleProduct{ - RoleID: r.ID, - ProductName: product, - } - - _, err = c.Post(roleProductURL, httpclient.SetBody(roleProductReq)) - if err != nil { - log.Errorf("CreateContributorRole failed, error: %v", err) - return err - } - permissionList := []string{ - permission.WorkflowTaskUUID, - permission.WorkflowListUUID, - permission.TestEnvDeleteUUID, - permission.TestEnvManageUUID, - permission.TestEnvListUUID, - permission.TestUpdateEnvUUID, - permission.BuildListUUID, - permission.TestListUUID, - permission.ServiceTemplateListUUID, - } - permissionReq := RolePermissionReq{ - RoleID: r.ID, - RoleType: r.RoleType, - PermissionUUID: permissionList, - ProductName: product, - } - rolePermissionURL := "/directory/rolePermission" - - _, err = c.Post(rolePermissionURL, httpclient.SetBody(permissionReq)) - if err != nil { - log.Errorf("CreateContributorRole failed, error: %v", err) - return err - } - - return nil -} - -// ListRoles 根据项目里面的角色名称获取对应的角色ID -func (c *Client) ListRoles(productName string, log *zap.SugaredLogger) (*Role, error) { - url := "/directory/roles" - qs := map[string]string{ - "name": AllUsersRoleName, - "productName": productName, - "roleType": strconv.Itoa(ProjectType), - } - - resp := make([]*Role, 0) - _, err := c.Get(url, httpclient.SetResult(&resp), httpclient.SetQueryParams(qs)) - if err != nil { - log.Errorf("ListRoles error: %v", err) - return nil, err - } - - if len(resp) == 0 { - return nil, fmt.Errorf("ListRoles Not Found") - } - - return resp[0], nil -} - -func (c *Client) ListUserEnvPermission(productName string, userID int, log *zap.SugaredLogger) ([]*UserEnv, error) { - url := "/directory/userEnvPermission" - qs := map[string]string{ - "productName": productName, - "userId": strconv.Itoa(userID), - } - - resp := make([]*UserEnv, 0) - _, err := c.Get(url, httpclient.SetResult(&resp), httpclient.SetQueryParams(qs)) - if err != nil { - log.Errorf("ListUserEnvPermission error: %v", err) - return nil, err - } - - return resp, nil -} - -func (c *Client) DeleteUserEnvPermission(productName, username string, userID int, log *zap.SugaredLogger) error { - url := "/directory/userEnvPermission" - qs := map[string]string{ - "productName": productName, - "envName": strings.ToLower(username), - "userId": strconv.Itoa(userID), - } - - _, err := c.Delete(url, httpclient.SetQueryParams(qs)) - if err != nil { - log.Errorf("DeleteUserEnvPermission error: %v", err) - return err - } - - return nil -} - -func (c *Client) GetUserEnvPermission(userID int, log *zap.SugaredLogger) ([]*UserEnv, error) { - url := "/directory/userEnvPermission" - - resp := make([]*UserEnv, 0) - _, err := c.Get(url, httpclient.SetResult(&resp), httpclient.SetQueryParam("userId", strconv.Itoa(userID))) - if err != nil { - log.Errorf("UserEnvPermission request error: %v", err) - return nil, err - } - - return resp, nil -} diff --git a/pkg/tool/codehub/client.go b/pkg/tool/codehub/client.go index cc25380e4a53653d18b218947e45496162dc685a..db736c98eef8fd0fcca53691281cbb58718b54bb 100644 --- a/pkg/tool/codehub/client.go +++ b/pkg/tool/codehub/client.go @@ -22,26 +22,32 @@ import ( "io" "io/ioutil" "net/http" + "net/url" "github.com/koderover/zadig/pkg/tool/log" ) type CodeHubClient struct { - AK string `json:"ak"` - SK string `json:"sk"` - Region string `json:"region"` + AK string `json:"ak"` + SK string `json:"sk"` + Region string `json:"region"` + ProxyAddr string `json:"proxy_addr"` + EnableProxy bool `json:"enable_proxy"` } -func NewCodeHubClient(ak, sk, region string) *CodeHubClient { +func NewCodeHubClient(ak, sk, region, proxyAddr string, enableProxy bool) *CodeHubClient { return &CodeHubClient{ - AK: ak, - SK: sk, - Region: region, + AK: ak, + SK: sk, + Region: region, + ProxyAddr: proxyAddr, + EnableProxy: enableProxy, } } // Just apply the signature and request the CodeHub interface func (c *CodeHubClient) sendRequest(method, path string, payload []byte) (io.ReadCloser, error) { + var client *http.Client r, err := http.NewRequest(method, fmt.Sprintf("%s.%s.%s%s", "https://codehub-ext", c.Region, "myhuaweicloud.com", path), ioutil.NopCloser(bytes.NewBuffer(payload))) if r == nil || err != nil { log.Errorf("http.NewRequest error:%s", err) @@ -57,7 +63,17 @@ func (c *CodeHubClient) sendRequest(method, path string, payload []byte) (io.Rea log.Errorf("signer.Sign error:%s", err) return nil, err } - resp, err := http.DefaultClient.Do(r) + if c.EnableProxy { + proxyURL, err := url.Parse(c.ProxyAddr) + if err != nil { + return nil, err + } + transport := &http.Transport{Proxy: http.ProxyURL(proxyURL)} + client = &http.Client{Transport: transport} + } else { + client = http.DefaultClient + } + resp, err := client.Do(r) if err != nil || resp == nil { log.Errorf("http.DefaultClient.Do error:%s", err) return nil, err diff --git a/pkg/tool/crypto/aes.go b/pkg/tool/crypto/aes.go index 65e69ce07140e5f084114efa774544299f7fc4de..1084521a1fb71eb9be328b224e17973519b3075f 100644 --- a/pkg/tool/crypto/aes.go +++ b/pkg/tool/crypto/aes.go @@ -51,6 +51,10 @@ func getAESKey() string { return aesKey } +func GetAesKey() string { + return getAESKey() +} + func AesEncrypt(src string) (string, error) { client, err := NewAes(getAESKey()) if err != nil { @@ -63,8 +67,28 @@ func AesEncrypt(src string) (string, error) { return dest, nil } -func AesDecrypt(src string) (string, error) { - client, err := NewAes(getAESKey()) +func AesEncryptByKey(src, aesKey string) (string, error) { + client, err := NewAes(aesKey) + if err != nil { + return "", err + } + dest, err := client.Encrypt(src) + if err != nil { + return "", err + } + return dest, nil +} + +func AesDecrypt(src string, aesKey ...string) (string, error) { + var ( + err error + client *Aes + ) + if len(aesKey) > 0 { + client, err = NewAes(aesKey[0]) + } else { + client, err = NewAes(getAESKey()) + } if err != nil { return "", err } diff --git a/pkg/tool/dockerhost/docker_host.go b/pkg/tool/dockerhost/docker_host.go new file mode 100644 index 0000000000000000000000000000000000000000..d6da5d435752f8b77c18b9d27adbe1ce351f27f2 --- /dev/null +++ b/pkg/tool/dockerhost/docker_host.go @@ -0,0 +1,201 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dockerhost + +import ( + "context" + "fmt" + "math/rand" + "sync" + "time" + + "github.com/buraksezer/consistent" + "github.com/cespare/xxhash" + "go.uber.org/zap" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" +) + +var once sync.Once + +type Member string + +func (m Member) String() string { + return string(m) +} + +type hasher struct{} + +func (h hasher) Sum64(data []byte) uint64 { + return xxhash.Sum64(data) +} + +type ClusterID string + +type DockerHostsI interface { + GetBestHost(ClusterID, string) string + + Sync() +} + +type dockerhosts struct { + rwLock *sync.RWMutex + store map[ClusterID]*consistent.Consistent + hubServerAddr string + syncInterval time.Duration + + logger *zap.SugaredLogger +} + +var dockerHosts DockerHostsI + +func init() { + rand.Seed(time.Now().UTC().UnixNano()) +} + +func NewDockerHosts(hubServerAddr string, logger *zap.SugaredLogger) DockerHostsI { + once.Do(func() { + dockerHosts = &dockerhosts{ + rwLock: &sync.RWMutex{}, + store: map[ClusterID]*consistent.Consistent{}, + hubServerAddr: hubServerAddr, + logger: logger, + } + + go dockerHosts.Sync() + }) + + return dockerHosts +} + +func (d *dockerhosts) GetBestHost(clusterID ClusterID, key string) string { + if d.store[clusterID] == nil { + d.initClusterInfo(clusterID) + } + + member := d.store[clusterID].LocateKey([]byte(key)) + + return member.String() +} + +func (d *dockerhosts) initClusterInfo(clusterID ClusterID) { + d.rwLock.Lock() + defer d.rwLock.Unlock() + + members := d.getDockerHostsSvc(clusterID) + + cfg := consistent.Config{ + PartitionCount: 271, + ReplicationFactor: 20, + Load: 1.25, + Hasher: hasher{}, + } + + d.store[clusterID] = consistent.New(members, cfg) +} + +func (d *dockerhosts) getDockerHostsSvc(clusterID ClusterID) []consistent.Member { + ns := config.Namespace() + if string(clusterID) != setting.LocalClusterID { + ns = setting.AttachedClusterNamespace + } + + kclient, err := kubeclient.GetKubeClient(d.hubServerAddr, string(clusterID)) + if err != nil { + d.logger.Warnf("Failed to get kubeclient for cluster %q: %s. Try to use default dockerhosts.", clusterID, err) + return d.getDefaultDockerHosts() + } + + dindSts := &appsv1.StatefulSet{} + err = kclient.Get(context.TODO(), client.ObjectKey{ + Name: "dind", + Namespace: ns, + }, dindSts) + if err != nil { + d.logger.Warnf("Failed to get dind statefuleset in namespace %q of cluster %q: %s", ns, clusterID, err) + return d.getDefaultDockerHosts() + } + + members := []consistent.Member{} + for i := 0; i < int(*dindSts.Spec.Replicas); i++ { + members = append(members, d.genDindAddr(i)) + } + + return members +} + +func (d *dockerhosts) getDefaultDockerHosts() []consistent.Member { + return []consistent.Member{d.genDindAddr(0)} +} + +func (d *dockerhosts) genDindAddr(idx int) Member { + return Member(fmt.Sprintf("tcp://dind-%d.dind:2375", idx)) +} + +func (d *dockerhosts) Sync() { + d.logger.Info("Begin to sync") + + wait.Forever(func() { + d.rwLock.Lock() + defer d.rwLock.Unlock() + + for clusterID, consistentHash := range d.store { + currentMembers := d.getDockerHostsSvc(clusterID) + oldMembers := consistentHash.GetMembers() + + d.logger.Infof("Cluster: %q. Current Members: %d. Old Members: %d", clusterID, len(currentMembers), len(oldMembers)) + + addedMembers, deletedMembers := d.diffMembers(oldMembers, currentMembers) + for _, member := range addedMembers { + consistentHash.Add(member) + } + for _, member := range deletedMembers { + consistentHash.Remove(member.String()) + } + } + }, 3*time.Minute) +} + +func (d *dockerhosts) diffMembers(old, current []consistent.Member) (added, deleted []consistent.Member) { + curMap := make(map[consistent.Member]struct{}, len(current)) + for _, member := range current { + curMap[member] = struct{}{} + } + + oldMap := make(map[consistent.Member]struct{}, len(old)) + for _, member := range old { + oldMap[member] = struct{}{} + } + + for _, member := range old { + if _, found := curMap[member]; !found { + deleted = append(deleted, member) + } + } + for _, member := range current { + if _, found := oldMap[member]; !found { + added = append(added, member) + } + } + + return added, deleted +} diff --git a/pkg/tool/dockerhost/dockerhosts_test.go b/pkg/tool/dockerhost/dockerhosts_test.go new file mode 100644 index 0000000000000000000000000000000000000000..6df1f4a3b7ea49082815b644b22debf30526ccaa --- /dev/null +++ b/pkg/tool/dockerhost/dockerhosts_test.go @@ -0,0 +1,50 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dockerhost + +import ( + "math/rand" + "testing" + "time" + + "go.uber.org/zap" +) + +func init() { + rand.Seed(time.Now().UTC().UnixNano()) +} + +func TestNewDockerHosts(t *testing.T) { + hubServerAddr := "hub-server.zadig" + logger, _ := zap.NewProduction() + dockerhosts := NewDockerHosts(hubServerAddr, logger.Sugar()) + + clusters := []ClusterID{ClusterID("local"), ClusterID("worker0"), ClusterID("worker1")} + svcs := []string{"svc0", "svc1", "svc2", "svc3", "svc4"} + + for i := 0; i < len(clusters); i++ { + t.Logf("In Cluster %q\n", clusters[i]) + + for j := 0; j < 10; j++ { + svc := svcs[rand.Intn(len(svcs))] + host := dockerhosts.GetBestHost(clusters[i], svc) + t.Logf("\tHost for svc %q: %q\n", svc, host) + } + + t.Log("\n") + } +} diff --git a/pkg/tool/errors/const.go b/pkg/tool/errors/const.go index 0a42fc138728ec2a7a05f6f0bf43dfd9aef35473..744d01f80817b6084eba3b6c1d37c941b28e7b2a 100644 --- a/pkg/tool/errors/const.go +++ b/pkg/tool/errors/const.go @@ -70,7 +70,7 @@ const ( const ( FindProductTmplErrMsg = "获取产品失败" - FindProductServiceErrMsg = "产品不包含任何服务" + FindProductServiceErrMsg = "不包含任何服务" ) // Service & Containers @@ -117,3 +117,7 @@ const ( DuplicateClusterNameFound = "已经有相同名称的集群存在" StartPodTimeout = "这些服务中可能存在未正常运行的实例" ) + +const ( + DeleteVirtualServiceErrMsg = "删除 VirtualService 失败" +) diff --git a/pkg/tool/errors/http_errors.go b/pkg/tool/errors/http_errors.go index 38c3a8ac28f58c6b24a8492811ac267cade4c322..81b8a4ff06f336d8fef59a4dbe951336f551a9c9 100644 --- a/pkg/tool/errors/http_errors.go +++ b/pkg/tool/errors/http_errors.go @@ -44,7 +44,8 @@ var ( ErrListUsers = NewHTTPError(6002, "列出用户信息失败") // ErrFindUser ... ErrFindUser = NewHTTPError(6002, "获取用户信息失败") - + // ErrCallBackUser ... + ErrCallBackUser = NewHTTPError(6003, "dex回调用户失败") //----------------------------------------------------------------------------------------------- // Team APIs Range: 6020 - 6039 //----------------------------------------------------------------------------------------------- @@ -134,7 +135,7 @@ var ( // ErrGetProductRevision ... ErrGetProductRevision = NewHTTPError(6067, "获取产品版本失败") // ErrFindProduct ... - ErrFindProduct = NewHTTPError(6068, "获取集成环境失败") + ErrFindProduct = NewHTTPError(6068, "获取环境失败") // ErrGetProductAuth ... ErrGetProductAuth = NewHTTPError(6069, "获取产品权限失败") // ErrUpdateProductAuth ... @@ -144,15 +145,15 @@ var ( // ErrStopPatchProduct ... ErrStopPatchProduct = NewHTTPError(6072, "产品收集集成测试覆盖率失败") // ErrCreateEnv ... - ErrCreateEnv = NewHTTPError(6074, "创建集成环境失败") + ErrCreateEnv = NewHTTPError(6074, "创建环境失败") // ErrListEnvs ... - ErrListEnvs = NewHTTPError(6075, "列出集成环境失败") + ErrListEnvs = NewHTTPError(6075, "列出环境失败") // ErrUpdateEnv ... - ErrUpdateEnv = NewHTTPError(6076, "更新集成环境失败") + ErrUpdateEnv = NewHTTPError(6076, "更新环境失败") // ErrDeleteEnv ... - ErrDeleteEnv = NewHTTPError(6077, "删除集成环境失败") + ErrDeleteEnv = NewHTTPError(6077, "删除环境失败") // ErrGetEnv ... - ErrGetEnv = NewHTTPError(6078, "获取集成环境失败") + ErrGetEnv = NewHTTPError(6078, "获取环境失败") // ErrFindProductTmpl ... ErrFindProductTmpl = NewHTTPError(6079, "项目已删除，环境正在回收中") // TODO: max error code reached, sharing error code with create product @@ -192,6 +193,16 @@ var ( ErrUpdateService = NewHTTPError(6092, "更新服务失败") // ErrListPodEvents ... ErrListPodEvents = NewHTTPError(6093, "列出服务事件失败") + //ErrListResources ... + ErrListResources = NewHTTPError(6094, "列出对象资源失败") + //ErrUpdateResource ... + ErrUpdateResource = NewHTTPError(6095, "更新对象资源失败") + //ErrDeleteResource ... + ErrDeleteResource = NewHTTPError(6096, "删除对象资源失败") + // ErrLoginPm ... + ErrLoginPm = NewHTTPError(6099, "登录主机失败") + + ErrDeleteSvcHasSvcsInSubEnv = NewHTTPError(6094, "删除服务失败，待删除服务存在于子环境中") //----------------------------------------------------------------------------------------------- // it report APIs Range: 6100 - 6199 @@ -273,6 +284,9 @@ var ( ErrCreateTaskFailed = NewHTTPError(6168, "创建工作流任务失败") + // ErrApproveTask ... + ErrApproveTask = NewHTTPError(6169, "批准工作流任务失败") + //----------------------------------------------------------------------------------------------- // Keystore APIs Range: 6180 - 6189 //----------------------------------------------------------------------------------------------- @@ -454,6 +468,10 @@ var ( ErrDeleteTestModule = NewHTTPError(6533, "删除测试模块失败") // ErrGetTestReport ... ErrGetTestReport = NewHTTPError(6534, "获取html测试报告失败") + // ErrCreateScanningModule ... + ErrCreateScanningModule = NewHTTPError(6535, "新建扫描模块失败") + // ErrCreateScanningModule ... + ErrUpdateScanningModule = NewHTTPError(6536, "更新扫描模块失败") // Workflow APIs Range: 6540 - 6550 //----------------------------------------------------------------------------------------------- @@ -583,6 +601,7 @@ var ( ErrListPrivateKeys = NewHTTPError(6684, "列出私钥失败") ErrDeletePrivateKey = NewHTTPError(6685, "删除私钥失败") ErrDeleteUsedPrivateKey = NewHTTPError(6686, "删除私钥失败，此私钥已经被引用，请确认") + ErrBulkCreatePrivateKey = NewHTTPError(6687, "批量创建私钥失败") //----------------------------------------------------------------------------------------------- // signature APIs Range: 6690 - 6699 @@ -670,4 +689,25 @@ var ( ErrTestJenkinsConnection = NewHTTPError(6835, "用户名或者密码不正确") ErrListJobNames = NewHTTPError(6836, "获取job名称列表失败") ErrListJobBuildArgs = NewHTTPError(6837, "获取job构建参数列表失败") + + //----------------------------------------------------------------------------------------------- + // external link Error Range: 6840 - 6849 + //----------------------------------------------------------------------------------------------- + ErrCreateExternalLink = NewHTTPError(6841, "创建链接失败") + ErrUpdateExternalLink = NewHTTPError(6842, "更新链接失败") + ErrDeleteExternalLink = NewHTTPError(6843, "删除链接失败") + ErrListExternalLink = NewHTTPError(6844, "获取链接列表失败") + + //----------------------------------------------------------------------------------------------- + // helm releated Error Range: 6850 - 6869 + //----------------------------------------------------------------------------------------------- + ErrListHelmReleases = NewHTTPError(6850, "获取release失败") + ErrGetHelmCharts = NewHTTPError(6851, "获取chart信息失败") + + //----------------------------------------------------------------------------------------------- + // plugins releated Error Range: 6870 - 6879 + //----------------------------------------------------------------------------------------------- + ErrListPluginRepo = NewHTTPError(6870, "获取plugin仓库失败") + ErrUpsertPluginRepo = NewHTTPError(6871, "更新plugin仓库失败") + ErrDeletePluginRepo = NewHTTPError(6872, "删除plugin仓库失败") ) diff --git a/pkg/tool/gerrit/client.go b/pkg/tool/gerrit/client.go index 74d3779a6d9b3a5b30943dfd3d027f681ef2a518..03337e30ac1328f066fbcc0560c102ea1db32de7 100644 --- a/pkg/tool/gerrit/client.go +++ b/pkg/tool/gerrit/client.go @@ -18,10 +18,12 @@ package gerrit import ( "fmt" + "net" "net/http" "net/url" "regexp" "strings" + "time" gerrit "github.com/andygrunwald/go-gerrit" ) @@ -32,8 +34,14 @@ type Client struct { cli *gerrit.Client } -func NewClient(address, accessToken string) *Client { - httpClient := &http.Client{Transport: &BasicAuthTransporter{EncodedUserPass: accessToken}} +func NewClient(address, accessToken, proxyAddr string, enableProxy bool) *Client { + httpClient := &http.Client{ + Transport: &BasicAuthTransporter{ + EncodedUserPass: accessToken, + ProxyAddr: proxyAddr, + EnableProxy: enableProxy, + }, + } cli, _ := gerrit.NewClient(address+"/a", httpClient) return &Client{cli: cli} } @@ -42,6 +50,22 @@ func (c *Client) ListProjects() ([]*gerrit.ProjectInfo, error) { return c.ListProjectsByKey("") } +// convert keyword like `hello world` to `.*hello.*world.*` +func keywordToRegexp(keyword string) string { + splits := strings.Split(keyword, " ") + regSlices := make([]string, 0, len(splits)) + for _, split := range splits { + s := strings.TrimSpace(split) + if s != "" { + regSlices = append(regSlices, regexp.QuoteMeta(s)) + } + } + if len(regSlices) == 0 { + return ".*" + } + return ".*" + strings.Join(regSlices, ".*") + ".*" +} + func (c *Client) ListProjectsByKey(keyword string) ([]*gerrit.ProjectInfo, error) { opts := &gerrit.ProjectOptions{ Tree: false, @@ -51,8 +75,13 @@ func (c *Client) ListProjectsByKey(keyword string) ([]*gerrit.ProjectInfo, error }, } + // query with substring if no space included, otherwise use regexp if keyword != "" { - opts.Substring = keyword + if !strings.Contains(keyword, " ") { + opts.Substring = keyword + } else { + opts.Regex = keywordToRegexp(keyword) + } } resp, _, err := c.cli.Projects.ListProjects(opts) @@ -241,12 +270,32 @@ func (c *Client) CompareTwoPatchset(changeID, newPatchSetID, oldPatchSetID strin type BasicAuthTransporter struct { EncodedUserPass string + ProxyAddr string + EnableProxy bool } func (bt *BasicAuthTransporter) RoundTrip(req *http.Request) (*http.Response, error) { auth := "Basic " + bt.EncodedUserPass req.Header.Set("Authorization", auth) - return http.DefaultTransport.RoundTrip(req) + transport := &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + if bt.EnableProxy { + proxyURL, err := url.Parse(bt.ProxyAddr) + if err != nil { + return nil, err + } + transport.Proxy = http.ProxyURL(proxyURL) + } + return transport.RoundTrip(req) } var backslash = regexp.MustCompile("%2[F|f]") diff --git a/pkg/tool/gerrit/client_test.go b/pkg/tool/gerrit/client_test.go new file mode 100644 index 0000000000000000000000000000000000000000..7c3c23fcd21e556899f0e97d9539120d623d117e --- /dev/null +++ b/pkg/tool/gerrit/client_test.go @@ -0,0 +1,42 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gerrit + +import "testing" + +func TestKeywordToRegexp(t *testing.T) { + type testcase struct { + keyword string + result string + } + testcases := []testcase{ + {"", ".*"}, + {" ", ".*"}, + {" ", ".*"}, + {"ab", ".*ab.*"}, + {" a b ", ".*a.*b.*"}, + {" a b c d ", ".*a.*b.*c.*d.*"}, + {" a/b c-d", ".*a/b.*c-d.*"}, + } + + for _, ts := range testcases { + regstr := keywordToRegexp(ts.keyword) + if regstr != ts.result { + t.Errorf("Expected result for keyword <%s> to be <%s> but got <%s>", ts.keyword, ts.result, regstr) + } + } +} diff --git a/pkg/tool/git/github/apps.go b/pkg/tool/git/github/apps.go index 5aaea100b067ddc00223e1b2c0455b7cf2c0bfa3..2b5af5e771e936e0901e5b3734ab87acc6c9be9d 100644 --- a/pkg/tool/git/github/apps.go +++ b/pkg/tool/git/github/apps.go @@ -32,6 +32,10 @@ func (c *Client) ListInstallations(ctx context.Context, opts *ListOptions) ([]*g return res, r, err }, opts)) + if err != nil { + return nil, err + } + var res []*github.Installation is, ok := installations.([]interface{}) if !ok { diff --git a/pkg/tool/git/github/organizations.go b/pkg/tool/git/github/organizations.go index 414785d1eba9652fa126533e88733535a29132f6..2caaaa2a89248481d491259051aa21b8f286ecfb 100644 --- a/pkg/tool/git/github/organizations.go +++ b/pkg/tool/git/github/organizations.go @@ -32,6 +32,10 @@ func (c *Client) ListOrganizationsForAuthenticatedUser(ctx context.Context, opts return res, r, err }, opts)) + if err != nil { + return nil, err + } + var res []*github.Organization os, ok := organizations.([]interface{}) if !ok { diff --git a/pkg/tool/git/github/pull_requests.go b/pkg/tool/git/github/pull_requests.go index 5170b73715d6bc2aaee6822b969659589d45ae20..39c0adeac8e1ff7e4ad68b56b7e744fb92a5d208 100644 --- a/pkg/tool/git/github/pull_requests.go +++ b/pkg/tool/git/github/pull_requests.go @@ -50,6 +50,10 @@ func (c *Client) ListCommits(ctx context.Context, owner string, repo string, num return res, r, err }, opts)) + if err != nil { + return nil, err + } + var res []*github.RepositoryCommit cs, ok := commits.([]interface{}) if !ok { @@ -72,6 +76,10 @@ func (c *Client) ListFiles(ctx context.Context, owner string, repo string, numbe return res, r, err }, opts)) + if err != nil { + return nil, err + } + var res []*github.CommitFile fs, ok := files.([]interface{}) if !ok { diff --git a/pkg/tool/git/github/repositories.go b/pkg/tool/git/github/repositories.go index 77f4e7ce0a5e35f588464b0bee8daecc409d3bb3..84ce6aa3e7ab3d8cccd251125c27ec485ac95961 100644 --- a/pkg/tool/git/github/repositories.go +++ b/pkg/tool/git/github/repositories.go @@ -29,8 +29,9 @@ import ( fsutil "github.com/koderover/zadig/pkg/util/fs" ) -func (c *Client) ListRepositoriesForAuthenticatedUser(ctx context.Context, opts *ListOptions) ([]*github.Repository, error) { +func (c *Client) ListRepositoriesForAuthenticatedUser(ctx context.Context, user string, opts *ListOptions) ([]*github.Repository, error) { repositories, err := wrap(paginated(func(o *github.ListOptions) ([]interface{}, *github.Response, error) { + // Note. parameter user is not used when list repositories because private repos will NOT be in response data from gitHub even user is the exact owner of these private repos rs, r, err := c.Repositories.List(ctx, "", &github.RepositoryListOptions{ListOptions: *o}) var res []interface{} for _, r := range rs { @@ -39,6 +40,10 @@ func (c *Client) ListRepositoriesForAuthenticatedUser(ctx context.Context, opts return res, r, err }, opts)) + if err != nil { + return nil, err + } + var res []*github.Repository rs, ok := repositories.([]interface{}) if !ok { @@ -61,6 +66,10 @@ func (c *Client) ListBranches(ctx context.Context, owner, repo string, opts *Lis return res, r, err }, opts)) + if err != nil { + return nil, err + } + var res []*github.Branch bs, ok := branches.([]interface{}) if !ok { @@ -83,6 +92,10 @@ func (c *Client) ListTags(ctx context.Context, owner, repo string, opts *ListOpt return res, r, err }, opts)) + if err != nil { + return nil, err + } + var res []*github.RepositoryTag ts, ok := tags.([]interface{}) if !ok { @@ -105,6 +118,10 @@ func (c *Client) ListHooks(ctx context.Context, owner, repo string, opts *ListOp return res, r, err }, opts)) + if err != nil { + return nil, err + } + var res []*github.Hook hs, ok := hooks.([]interface{}) if !ok { @@ -127,6 +144,10 @@ func (c *Client) ListReleases(ctx context.Context, owner, repo string, opts *Lis return res, r, err }, opts)) + if err != nil { + return nil, err + } + var res []*github.RepositoryRelease hs, ok := releases.([]interface{}) if !ok { @@ -149,6 +170,10 @@ func (c *Client) ListRepositoryCommits(ctx context.Context, owner, repo, path, b return res, r, err }, opts)) + if err != nil { + return nil, err + } + var res []*github.RepositoryCommit hs, ok := releases.([]interface{}) if !ok { @@ -185,11 +210,43 @@ func (c *Client) CreateHook(ctx context.Context, owner, repo string, hook *git.H Active: hook.Active, } created, err := wrap(c.Repositories.CreateHook(ctx, owner, repo, h)) - if h, ok := created.(*github.Hook); ok { - return h, err + if err != nil { + return nil, err } - return nil, err + res, ok := created.(*github.Hook) + if !ok { + return nil, fmt.Errorf("object is not a github Hook") + } + + return res, nil +} + +func (c *Client) UpdateHook(ctx context.Context, owner, repo string, id int64, hook *git.Hook) (*github.Hook, error) { + h := &github.Hook{ + Config: map[string]interface{}{ + "url": hook.URL, + "content_type": "json", + "secret": hook.Secret, + }, + } + if len(hook.Events) > 0 { + h.Events = hook.Events + } + if hook.Active != nil { + h.Active = hook.Active + } + updated, err := wrap(c.Repositories.EditHook(ctx, owner, repo, id, h)) + if err != nil { + return nil, err + } + + res, ok := updated.(*github.Hook) + if !ok { + return nil, fmt.Errorf("object is not a github Hook") + } + + return res, nil } func (c *Client) CreateStatus(ctx context.Context, owner, repo, ref string, status *github.RepoStatus) (*github.RepoStatus, error) { diff --git a/pkg/tool/git/gitlab/branch.go b/pkg/tool/git/gitlab/branch.go index 40763dd06aef74ac9a398fe106863a2b7c467d83..1c3ca13b0e56c9c3b6d23731070f6b2a2d35d6f1 100644 --- a/pkg/tool/git/gitlab/branch.go +++ b/pkg/tool/git/gitlab/branch.go @@ -19,9 +19,9 @@ package gitlab import "github.com/xanzy/go-gitlab" // ListBranches lists branches by projectID <- urlEncode(namespace/projectName) -func (c *Client) ListBranches(owner, repo string, opts *ListOptions) ([]*gitlab.Branch, error) { +func (c *Client) ListBranches(owner, repo, key string, opts *ListOptions) ([]*gitlab.Branch, error) { branches, err := wrap(paginated(func(o *gitlab.ListOptions) ([]interface{}, *gitlab.Response, error) { - bs, r, err := c.Branches.ListBranches(generateProjectName(owner, repo), &gitlab.ListBranchesOptions{ListOptions: *o}) + bs, r, err := c.Branches.ListBranches(generateProjectName(owner, repo), &gitlab.ListBranchesOptions{ListOptions: *o, Search: &key}) var res []interface{} for _, b := range bs { res = append(res, b) @@ -29,6 +29,10 @@ func (c *Client) ListBranches(owner, repo string, opts *ListOptions) ([]*gitlab. return res, r, err }, opts)) + if err != nil { + return nil, err + } + var res []*gitlab.Branch bs, ok := branches.([]interface{}) if !ok { diff --git a/pkg/tool/git/gitlab/client.go b/pkg/tool/git/gitlab/client.go index 2fe6053bc0f49dbd8d236a8f2252095c489d3903..b8213d82748f5dbb09459c53e630157f9c9a5fc1 100644 --- a/pkg/tool/git/gitlab/client.go +++ b/pkg/tool/git/gitlab/client.go @@ -19,6 +19,8 @@ package gitlab import ( "fmt" "io" + "net/http" + "net/url" "github.com/xanzy/go-gitlab" @@ -43,8 +45,25 @@ type Client struct { *gitlab.Client } -func NewClient(address, accessToken string) (*Client, error) { - cli, err := gitlab.NewOAuthClient(accessToken, gitlab.WithBaseURL(address)) +func NewClient(id int, address, accessToken, proxyAddr string, enableProxy bool) (*Client, error) { + var client *http.Client + if enableProxy { + proxyURL, err := url.Parse(proxyAddr) + if err != nil { + return nil, err + } + transport := &http.Transport{Proxy: http.ProxyURL(proxyURL)} + client = &http.Client{Transport: transport} + } else { + client = http.DefaultClient + } + + token, err := UpdateGitlabToken(id, accessToken) + if err != nil { + return nil, fmt.Errorf("failed to refresh gitlab token, err: %s", err) + } + + cli, err := gitlab.NewOAuthClient(token, gitlab.WithBaseURL(address), gitlab.WithHTTPClient(client)) if err != nil { return nil, fmt.Errorf("failed to create gitlab client, err: %s", err) } diff --git a/pkg/tool/git/gitlab/group_project.go b/pkg/tool/git/gitlab/group_project.go index 17ebe83b41f2e1cbb63e8ff45f99df8a53f15168..e3ad35907ce1abf1447024617be6f96c2daf3b17 100644 --- a/pkg/tool/git/gitlab/group_project.go +++ b/pkg/tool/git/gitlab/group_project.go @@ -43,6 +43,10 @@ func (c *Client) ListGroupProjects(namespace, keyword string, opts *ListOptions) return res, r, err }, opts)) + if err != nil { + return nil, err + } + var res []*gitlab.Project ps, ok := projects.([]interface{}) if !ok { diff --git a/pkg/tool/git/gitlab/merge_request.go b/pkg/tool/git/gitlab/merge_request.go index baa84c00ac3e59ed091f5601ec8a8d17ed969f19..2b3ad7f4d69d550a2af38c7a3c31f5b73e6ef076 100644 --- a/pkg/tool/git/gitlab/merge_request.go +++ b/pkg/tool/git/gitlab/merge_request.go @@ -18,13 +18,14 @@ package gitlab import "github.com/xanzy/go-gitlab" -func (c *Client) ListOpenedProjectMergeRequests(owner, repo, targetBranch string, opts *ListOptions) ([]*gitlab.MergeRequest, error) { +func (c *Client) ListOpenedProjectMergeRequests(owner, repo, targetBranch, key string, opts *ListOptions) ([]*gitlab.MergeRequest, error) { mergeRequests, err := wrap(paginated(func(o *gitlab.ListOptions) ([]interface{}, *gitlab.Response, error) { state := "opened" mopts := &gitlab.ListProjectMergeRequestsOptions{ State: &state, ListOptions: *o, + Search: &key, } if targetBranch != "" { mopts.TargetBranch = &targetBranch @@ -37,6 +38,10 @@ func (c *Client) ListOpenedProjectMergeRequests(owner, repo, targetBranch string return res, r, err }, opts)) + if err != nil { + return nil, err + } + var res []*gitlab.MergeRequest mrs, ok := mergeRequests.([]interface{}) if !ok { diff --git a/pkg/tool/git/gitlab/namespace.go b/pkg/tool/git/gitlab/namespace.go index cb7f594693d741f4cbb14faf806cd505799f867f..9a148efb3da1b222546344aa19cc799a90d0b0bd 100644 --- a/pkg/tool/git/gitlab/namespace.go +++ b/pkg/tool/git/gitlab/namespace.go @@ -37,6 +37,10 @@ func (c *Client) ListNamespaces(keyword string, opts *ListOptions) ([]*gitlab.Na return res, r, err }, opts)) + if err != nil { + return nil, err + } + var res []*gitlab.Namespace ns, ok := namespaces.([]interface{}) if !ok { diff --git a/pkg/tool/git/gitlab/project.go b/pkg/tool/git/gitlab/project.go index bea1ea800bf06d42a1144be34d49e45ef9035584..5e81a204c5e7f4f6493ba0490187fa1bfcba4a10 100644 --- a/pkg/tool/git/gitlab/project.go +++ b/pkg/tool/git/gitlab/project.go @@ -17,6 +17,7 @@ limitations under the License. package gitlab import ( + "fmt" "strings" "github.com/xanzy/go-gitlab" @@ -44,6 +45,10 @@ func (c *Client) ListUserProjects(owner, keyword string, opts *ListOptions) ([]* return res, r, err }, opts)) + if err != nil { + return nil, err + } + var res []*gitlab.Project ps, ok := projects.([]interface{}) if !ok { @@ -67,11 +72,34 @@ func (c *Client) AddProjectHook(owner, repo string, hook *git.Hook) (*gitlab.Pro return nil, err } - if h, ok := created.(*gitlab.ProjectHook); ok { - return h, nil + res, ok := created.(*gitlab.ProjectHook) + if !ok { + return nil, fmt.Errorf("object is not a gitlab Hook") } - return nil, err + return res, nil +} + +func (c *Client) UpdateProjectHook(owner, repo string, id int, hook *git.Hook) (*gitlab.ProjectHook, error) { + opts := &gitlab.EditProjectHookOptions{} + if hook.URL != "" { + opts.URL = gitlab.String(hook.URL) + } + if hook.Secret != "" { + opts.Token = gitlab.String(hook.Secret) + } + opts = addEventsToProjectHookEditOptions(hook.Events, opts) + updated, err := wrap(c.Projects.EditProjectHook(generateProjectName(owner, repo), id, opts)) + if err != nil { + return nil, err + } + + res, ok := updated.(*gitlab.ProjectHook) + if !ok { + return nil, fmt.Errorf("object is not a gitlab Hook") + } + + return res, nil } func (c *Client) DeleteProjectHook(owner, repo string, id int) error { @@ -92,6 +120,10 @@ func (c *Client) ListProjectHooks(owner, repo string, opts *ListOptions) ([]*git return res, r, err }, opts)) + if err != nil { + return nil, err + } + var res []*gitlab.ProjectHook hs, ok := hooks.([]interface{}) if !ok { @@ -140,3 +172,18 @@ func addEventsToProjectHookOptions(events []string, opts *gitlab.AddProjectHookO return opts } + +func addEventsToProjectHookEditOptions(events []string, opts *gitlab.EditProjectHookOptions) *gitlab.EditProjectHookOptions { + for _, evt := range events { + switch evt { + case git.PushEvent: + opts.PushEvents = boolptr.True() + case git.PullRequestEvent: + opts.MergeRequestsEvents = boolptr.True() + case git.BranchOrTagCreateEvent: + opts.TagPushEvents = boolptr.True() + } + } + + return opts +} diff --git a/pkg/tool/git/gitlab/repo.go b/pkg/tool/git/gitlab/repo.go index afb13dd2106f74b61822f113244a0764ac199367..ace765facfaebb7047d66b046284d828ee7ed52b 100644 --- a/pkg/tool/git/gitlab/repo.go +++ b/pkg/tool/git/gitlab/repo.go @@ -44,6 +44,10 @@ func (c *Client) ListTree(owner, repo, path, branch string, recursive bool, opts return res, r, err }, opts)) + if err != nil { + return nil, err + } + var res []*gitlab.TreeNode ns, ok := nodes.([]interface{}) if !ok { diff --git a/pkg/tool/git/gitlab/tag.go b/pkg/tool/git/gitlab/tag.go index 0ffb86e5fbdd50f73653f4ee33128fef2441ebbf..070013f6585dbb856b4154c73f92a00f9b1c78c6 100644 --- a/pkg/tool/git/gitlab/tag.go +++ b/pkg/tool/git/gitlab/tag.go @@ -19,9 +19,9 @@ package gitlab import "github.com/xanzy/go-gitlab" // ListTags lists branches by projectID <- urlEncode(namespace/projectName) -func (c *Client) ListTags(owner, repo string, opts *ListOptions) ([]*gitlab.Tag, error) { +func (c *Client) ListTags(owner, repo string, opts *ListOptions, key string) ([]*gitlab.Tag, error) { tags, err := wrap(paginated(func(o *gitlab.ListOptions) ([]interface{}, *gitlab.Response, error) { - ts, r, err := c.Tags.ListTags(generateProjectName(owner, repo), &gitlab.ListTagsOptions{ListOptions: *o}) + ts, r, err := c.Tags.ListTags(generateProjectName(owner, repo), &gitlab.ListTagsOptions{ListOptions: *o, Search: &key}) var res []interface{} for _, t := range ts { res = append(res, t) @@ -29,6 +29,10 @@ func (c *Client) ListTags(owner, repo string, opts *ListOptions) ([]*gitlab.Tag, return res, r, err }, opts)) + if err != nil { + return nil, err + } + var res []*gitlab.Tag ts, ok := tags.([]interface{}) if !ok { diff --git a/pkg/tool/git/gitlab/token.go b/pkg/tool/git/gitlab/token.go new file mode 100644 index 0000000000000000000000000000000000000000..3aad5069538057c0c00c02acc576feada0628870 --- /dev/null +++ b/pkg/tool/git/gitlab/token.go @@ -0,0 +1,103 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gitlab + +import ( + "fmt" + "sync" + "time" + + "github.com/koderover/zadig/pkg/shared/client/systemconfig" + "github.com/koderover/zadig/pkg/tool/httpclient" + "github.com/koderover/zadig/pkg/tool/log" +) + +type AccessToken struct { + AccessToken string `json:"access_token"` + TokenType string `json:"token_type"` + ExpiresIn int `json:"expires_in"` + RefreshToken string `json:"refresh_token"` + Scope string `json:"scope"` + CreatedAt int `json:"created_at"` +} + +var CodeHostLockMap sync.Map + +const TokenExpirationThreshold int64 = 7000 + +func UpdateGitlabToken(id int, accessToken string) (string, error) { + // if accessToken is empty, then it is either of ssh token type or username/password, we return empty + if accessToken == "" { + return "", nil + } + + lockInterface, _ := CodeHostLockMap.LoadOrStore(id, &sync.RWMutex{}) + lock := lockInterface.(*sync.RWMutex) + lock.Lock() + defer lock.Unlock() + + ch, err := systemconfig.New().GetCodeHost(id) + + if err != nil { + return "", fmt.Errorf("get codehost info error: [%s]", err) + } + + if time.Now().Unix()-ch.UpdatedAt <= TokenExpirationThreshold { + return ch.AccessToken, nil + } + + log.Infof("Starting to refresh gitlab token") + + token, err := refreshAccessToken(ch.Address, ch.AccessKey, ch.SecretKey, ch.RefreshToken) + if err != nil { + return "", err + } + + ch.AccessToken = token.AccessToken + ch.RefreshToken = token.RefreshToken + ch.UpdatedAt = int64(token.CreatedAt) + + // Since the new token is valid, we simply log the error + // No error will be returned, only the new token is returned + if err = systemconfig.New().UpdateCodeHost(ch.ID, ch); err != nil { + log.Errorf("failed to update codehost, err: %s", err) + } + + log.Infof("gitlab token for client [%d] changed from [%s] to [%s]", id, accessToken, token.AccessToken) + + return token.AccessToken, nil +} + +func refreshAccessToken(address, clientID, clientSecret, refreshToken string) (*AccessToken, error) { + httpClient := httpclient.New( + httpclient.SetHostURL(address), + ) + url := "/oauth/token" + queryParams := make(map[string]string) + queryParams["grant_type"] = "refresh_token" + queryParams["refresh_token"] = refreshToken + queryParams["client_id"] = clientID + queryParams["client_secret"] = clientSecret + + var accessToken *AccessToken + _, err := httpClient.Post(url, httpclient.SetQueryParams(queryParams), httpclient.SetResult(&accessToken)) + if err != nil { + return nil, err + } + + return accessToken, nil +} diff --git a/pkg/tool/gitee/branch.go b/pkg/tool/gitee/branch.go new file mode 100644 index 0000000000000000000000000000000000000000..2fb67b270edbddb1a4e6776610ed224726d00816 --- /dev/null +++ b/pkg/tool/gitee/branch.go @@ -0,0 +1,84 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gitee + +import ( + "context" + "fmt" + "time" + + "gitee.com/openeuler/go-gitee/gitee" + + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +func (c *Client) ListBranches(ctx context.Context, owner, repo string, opts *gitee.GetV5ReposOwnerRepoBranchesOpts) ([]gitee.Branch, error) { + bs, _, err := c.RepositoriesApi.GetV5ReposOwnerRepoBranches(ctx, owner, repo, opts) + if err != nil { + return nil, err + } + return bs, nil +} + +type Branch struct { + Name string `json:"name"` + Commit BranchCommit `json:"commit"` +} + +type BranchCommit struct { + Sha string `json:"sha"` + URL string `json:"url"` + Commit BranchCommmitInternalCommit `json:"commit"` +} + +type BranchCommmitInternalCommit struct { + Author BranchAuthor `json:"author"` + URL string `json:"url"` + Message string `json:"message"` + Tree BranchTree `json:"tree"` + Committer BranchCommitter `json:"committer"` +} + +type BranchAuthor struct { + Name string `json:"name"` + Date time.Time `json:"date"` + Email string `json:"email"` +} + +type BranchTree struct { + Sha string `json:"sha"` + URL string `json:"url"` +} + +type BranchCommitter struct { + Name string `json:"name"` + Date time.Time `json:"date"` + Email string `json:"email"` +} + +func (c *Client) GetSingleBranch(accessToken, owner, repo, branch string) (*Branch, error) { + httpClient := httpclient.New( + httpclient.SetHostURL(GiteeHOSTURL), + ) + url := fmt.Sprintf("/v5/repos/%s/%s/branches/%s", owner, repo, branch) + var branchInfo *Branch + _, err := httpClient.Get(url, httpclient.SetQueryParam("access_token", accessToken), httpclient.SetResult(&branchInfo)) + if err != nil { + return nil, err + } + return branchInfo, nil +} diff --git a/pkg/tool/gitee/client.go b/pkg/tool/gitee/client.go new file mode 100644 index 0000000000000000000000000000000000000000..f643b1de1254a346f98038efa0f355d9c6e5c59b --- /dev/null +++ b/pkg/tool/gitee/client.go @@ -0,0 +1,88 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gitee + +import ( + "context" + "net/http" + "net/url" + "time" + + "gitee.com/openeuler/go-gitee/gitee" + "golang.org/x/oauth2" + + "github.com/koderover/zadig/pkg/shared/client/systemconfig" + "github.com/koderover/zadig/pkg/tool/log" +) + +type Client struct { + *gitee.APIClient +} + +func NewClient(id int, accessToken, proxyAddr string, enableProxy bool) *Client { + var ( + client *gitee.APIClient + HttpClient *http.Client + ) + + conf := gitee.NewConfiguration() + dc := http.DefaultClient + if enableProxy { + p, err := url.Parse(proxyAddr) + if err == nil { + proxy := http.ProxyURL(p) + trans := &http.Transport{ + Proxy: proxy, + } + dc = &http.Client{Transport: trans} + } + } + + if accessToken != "" { + ch, err := systemconfig.New().GetCodeHost(id) + // The normal expiration time is 86400 + if err == nil && (time.Now().Unix()-ch.UpdatedAt) >= 86000 { + token, err := RefreshAccessToken(ch.RefreshToken) + if err == nil { + accessToken = token.AccessToken + ch.AccessToken = token.AccessToken + ch.RefreshToken = token.RefreshToken + ch.UpdatedAt = int64(token.CreatedAt) + + if err = systemconfig.New().UpdateCodeHost(ch.ID, ch); err != nil { + log.Errorf("failed to updateCodeHost err:%s", err) + } + } else { + log.Errorf("failed to refresh accessToken, err:%s", err) + } + } else if err != nil { + log.Errorf("failed to get codeHost id: %d, err:%s", id, err) + } + + ctx := context.WithValue(context.Background(), oauth2.HTTPClient, dc) + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: accessToken}, + ) + HttpClient = oauth2.NewClient(ctx, ts) + } else { + HttpClient = dc + } + conf.HTTPClient = HttpClient + client = gitee.NewAPIClient(conf) + + return &Client{client} +} diff --git a/pkg/tool/gitee/comment.go b/pkg/tool/gitee/comment.go new file mode 100644 index 0000000000000000000000000000000000000000..2292f53fce3ce9302bef086ef1c8688a08cd342a --- /dev/null +++ b/pkg/tool/gitee/comment.go @@ -0,0 +1,38 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package gitee + +import ( + "context" + + "gitee.com/openeuler/go-gitee/gitee" +) + +func (c *Client) CreateMergeRequestComment(ctx context.Context, owner, repo string, number int32, pullRequestCommentPostParam gitee.PullRequestCommentPostParam) (gitee.PullRequestComments, error) { + pullRequestComments, _, err := c.PullRequestsApi.PostV5ReposOwnerRepoPullsNumberComments(ctx, owner, repo, number, pullRequestCommentPostParam) + if err != nil { + return pullRequestComments, err + } + return pullRequestComments, nil +} + +func (c *Client) UpdateMergeRequestComment(ctx context.Context, owner, repo string, id int32, pullRequestCommentPatchParam gitee.PullRequestCommentPatchParam) error { + _, _, err := c.PullRequestsApi.PatchV5ReposOwnerRepoPullsCommentsId(ctx, owner, repo, id, pullRequestCommentPatchParam) + if err != nil { + return err + } + return nil +} diff --git a/pkg/tool/gitee/event_parsing.go b/pkg/tool/gitee/event_parsing.go new file mode 100644 index 0000000000000000000000000000000000000000..fbeef30a48ca51019bdd1a5bd0f5b77ce022ab20 --- /dev/null +++ b/pkg/tool/gitee/event_parsing.go @@ -0,0 +1,375 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gitee + +import ( + "encoding/json" + "fmt" + "net/http" + "time" +) + +// EventType represents a gitee event type. +type EventType string + +// List of available event types. +const ( + EventTypeMergeRequest EventType = "Merge Request Hook" + EventTypePush EventType = "Push Hook" + EventTypeTagPush EventType = "Tag Push Hook" +) + +const eventTypeHeader = "X-Gitee-Event" + +// HookEventType returns the event type for the given request. +func HookEventType(r *http.Request) EventType { + return EventType(r.Header.Get(eventTypeHeader)) +} + +func ParseHook(eventType EventType, payload []byte) (event interface{}, err error) { + return parseWebhook(eventType, payload) +} + +func parseWebhook(eventType EventType, payload []byte) (event interface{}, err error) { + switch eventType { + case EventTypeMergeRequest: + event = &PullRequestEvent{} + case EventTypePush: + event = &PushEvent{} + case EventTypeTagPush: + event = &TagPushEvent{} + default: + return nil, fmt.Errorf("unexpected event type: %s", eventType) + } + + if err := json.Unmarshal(payload, event); err != nil { + return nil, err + } + + return event, nil +} + +type TagPushEvent struct { + Ref string `json:"ref"` + Before string `json:"before"` + After string `json:"after"` + Created bool `json:"created"` + Deleted bool `json:"deleted"` + Compare string `json:"compare"` + Commits []TagPushEventCommit `json:"commits"` + HeadCommit TagPushEventHeadCommit `json:"head_commit"` + TotalCommitsCount int `json:"total_commits_count"` + CommitsMoreThanTen bool `json:"commits_more_than_ten"` + Repository EventRepo `json:"repository"` + Project TagPushEventProject `json:"project"` + UserID int `json:"user_id"` + UserName string `json:"user_name"` + User EventUser `json:"user"` + Pusher EventUser `json:"pusher"` + Sender EventUser `json:"sender"` + Enterprise EventEnterprise `json:"enterprise"` + HookName string `json:"hook_name"` + HookID int `json:"hook_id"` + HookURL string `json:"hook_url"` + Password string `json:"password"` + Timestamp interface{} `json:"timestamp"` + Sign string `json:"sign"` +} + +type TagPushEventCommit struct { + ID string `json:"id"` + TreeID string `json:"tree_id"` + ParentIds []string `json:"parent_ids"` + Distinct bool `json:"distinct"` + Message string `json:"message"` + Timestamp time.Time `json:"timestamp"` + URL string `json:"url"` + Author EventUser `json:"author"` + Committer EventUser `json:"committer"` + Added interface{} `json:"added"` + Removed interface{} `json:"removed"` + Modified []string `json:"modified"` +} + +type TagPushEventHeadCommit struct { + ID string `json:"id"` + TreeID string `json:"tree_id"` + ParentIds []string `json:"parent_ids"` + Distinct bool `json:"distinct"` + Message string `json:"message"` + Timestamp time.Time `json:"timestamp"` + URL string `json:"url"` + Author EventUser `json:"author"` + Committer EventUser `json:"committer"` + Added interface{} `json:"added"` + Removed interface{} `json:"removed"` + Modified []string `json:"modified"` +} + +type TagPushEventProject struct { + ID int `json:"id"` + Name string `json:"name"` + Path string `json:"path"` + FullName string `json:"full_name"` + Owner EventUser `json:"owner"` + Private bool `json:"private"` + HTMLURL string `json:"html_url"` + URL string `json:"url"` + Description string `json:"description"` + Fork bool `json:"fork"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + PushedAt time.Time `json:"pushed_at"` + GitURL string `json:"git_url"` + SSHURL string `json:"ssh_url"` + CloneURL string `json:"clone_url"` + SvnURL string `json:"svn_url"` + GitHTTPURL string `json:"git_http_url"` + GitSSHURL string `json:"git_ssh_url"` + GitSvnURL string `json:"git_svn_url"` + Homepage interface{} `json:"homepage"` + StargazersCount int `json:"stargazers_count"` + WatchersCount int `json:"watchers_count"` + ForksCount int `json:"forks_count"` + Language string `json:"language"` + HasIssues bool `json:"has_issues"` + HasWiki bool `json:"has_wiki"` + HasPages bool `json:"has_pages"` + License interface{} `json:"license"` + OpenIssuesCount int `json:"open_issues_count"` + DefaultBranch string `json:"default_branch"` + Namespace string `json:"namespace"` + NameWithNamespace string `json:"name_with_namespace"` + PathWithNamespace string `json:"path_with_namespace"` +} + +type PushEvent struct { + Ref string `json:"ref"` + Before string `json:"before"` + After string `json:"after"` + Created bool `json:"created"` + Deleted bool `json:"deleted"` + Compare string `json:"compare"` + Commits []PushEventCommit `json:"commits"` + TotalCommitsCount int `json:"total_commits_count"` + CommitsMoreThanTen bool `json:"commits_more_than_ten"` + Repository EventRepo `json:"repository"` + UserID int `json:"user_id"` + UserName string `json:"user_name"` + Pusher EventUser `json:"pusher"` + Enterprise EventEnterprise `json:"enterprise"` + HookName string `json:"hook_name"` + HookID int `json:"hook_id"` + HookURL string `json:"hook_url"` + Password string `json:"password"` + Timestamp interface{} `json:"timestamp"` + Sign string `json:"sign"` +} + +type PushEventCommit struct { + ID string `json:"id"` + TreeID string `json:"tree_id"` + ParentIds []string `json:"parent_ids"` + Distinct bool `json:"distinct"` + Message string `json:"message"` + Timestamp time.Time `json:"timestamp"` + URL string `json:"url"` + Author EventUser `json:"author"` + Committer EventUser `json:"committer"` + Added []string `json:"added,omitempty"` + Removed []string `json:"removed,omitempty"` + Modified []string `json:"modified,omitempty"` +} + +type PullRequestEvent struct { + Action string `json:"action"` + ActionDesc string `json:"action_desc"` + Project PullRequestEventProject `json:"project"` + PullRequest *PullRequestEventPullRequest `json:"pull_request"` + Number int `json:"number"` + Iid int `json:"iid"` + Title string `json:"title"` + Body interface{} `json:"body"` + Languages []string `json:"languages"` + State string `json:"state"` + MergeStatus string `json:"merge_status"` + MergeCommitSha string `json:"merge_commit_sha"` + URL string `json:"url"` + SourceBranch string `json:"source_branch"` + TargetBranch string `json:"target_branch"` + Repository PullRequestEventRepository `json:"repository"` + Author EventUser `json:"author"` + UpdatedBy EventUser `json:"updated_by"` + Sender EventUser `json:"sender"` + TargetUser EventUser `json:"target_user"` + Enterprise EventEnterprise `json:"enterprise"` + HookName string `json:"hook_name"` + HookID int `json:"hook_id"` + HookURL string `json:"hook_url"` + Password string `json:"password"` + Timestamp interface{} `json:"timestamp"` + Sign string `json:"sign"` +} + +type PullRequestEventProject struct { + ID int `json:"id"` + Language interface{} `json:"language"` + License interface{} `json:"license"` + Name string `json:"name"` + NameWithNamespace string `json:"name_with_namespace"` + Namespace string `json:"namespace"` + OpenIssuesCount int `json:"open_issues_count"` + Path string `json:"path"` + PathWithNamespace string `json:"path_with_namespace"` + Private bool `json:"private"` + PushedAt time.Time `json:"pushed_at"` + SSHURL string `json:"ssh_url"` + StargazersCount int `json:"stargazers_count"` + SvnURL string `json:"svn_url"` + UpdatedAt time.Time `json:"updated_at"` + URL string `json:"url"` + WatchersCount int `json:"watchers_count"` +} + +type PullRequestEventPullRequest struct { + ID int `json:"id"` + Number int `json:"number"` + State string `json:"state"` + HTMLURL string `json:"html_url"` + DiffURL string `json:"diff_url"` + PatchURL string `json:"patch_url"` + Title string `json:"title"` + Body interface{} `json:"body"` + Languages []string `json:"languages"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + ClosedAt interface{} `json:"closed_at"` + MergedAt interface{} `json:"merged_at"` + MergeCommitSha string `json:"merge_commit_sha"` + User EventUser `json:"user"` + Head *PullRequestEventBase `json:"head"` + Base PullRequestEventBase `json:"base"` + Merged bool `json:"merged"` + Mergeable interface{} `json:"mergeable"` + MergeStatus string `json:"merge_status"` + Comments int `json:"comments"` + Commits int `json:"commits"` + Additions int `json:"additions"` + Deletions int `json:"deletions"` + ChangedFiles int `json:"changed_files"` +} + +type PullRequestEventBase struct { + Label string `json:"label"` + Ref string `json:"ref"` + Sha string `json:"sha"` + User EventUser `json:"user"` + Repo EventRepo `json:"repo"` +} + +type EventRepo struct { + ID int `json:"id"` + Name string `json:"name"` + Path string `json:"path"` + FullName string `json:"full_name"` + Owner EventUser `json:"owner"` + Private bool `json:"private"` + HTMLURL string `json:"html_url"` + URL string `json:"url"` + Description string `json:"description"` + Fork bool `json:"fork"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + PushedAt time.Time `json:"pushed_at"` + GitURL string `json:"git_url"` + SSHURL string `json:"ssh_url"` + CloneURL string `json:"clone_url"` + SvnURL string `json:"svn_url"` + GitHTTPURL string `json:"git_http_url"` + GitSSHURL string `json:"git_ssh_url"` + GitSvnURL string `json:"git_svn_url"` + Homepage interface{} `json:"homepage"` + StargazersCount int `json:"stargazers_count"` + WatchersCount int `json:"watchers_count"` + ForksCount int `json:"forks_count"` + Language string `json:"language"` + HasIssues bool `json:"has_issues"` + HasWiki bool `json:"has_wiki"` + HasPages bool `json:"has_pages"` + License interface{} `json:"license"` + OpenIssuesCount int `json:"open_issues_count"` + DefaultBranch string `json:"default_branch"` + Namespace string `json:"namespace"` + NameWithNamespace string `json:"name_with_namespace"` + PathWithNamespace string `json:"path_with_namespace"` +} + +type PullRequestEventRepository struct { + ID int `json:"id"` + Name string `json:"name"` + Path string `json:"path"` + FullName string `json:"full_name"` + Owner EventUser `json:"owner"` + Private bool `json:"private"` + HTMLURL string `json:"html_url"` + URL string `json:"url"` + Description string `json:"description"` + Fork bool `json:"fork"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + PushedAt time.Time `json:"pushed_at"` + GitURL string `json:"git_url"` + SSHURL string `json:"ssh_url"` + CloneURL string `json:"clone_url"` + SvnURL string `json:"svn_url"` + GitHTTPURL string `json:"git_http_url"` + GitSSHURL string `json:"git_ssh_url"` + GitSvnURL string `json:"git_svn_url"` + Homepage interface{} `json:"homepage"` + StargazersCount int `json:"stargazers_count"` + WatchersCount int `json:"watchers_count"` + ForksCount int `json:"forks_count"` + Language string `json:"language"` + HasIssues bool `json:"has_issues"` + HasWiki bool `json:"has_wiki"` + HasPages bool `json:"has_pages"` + License interface{} `json:"license"` + OpenIssuesCount int `json:"open_issues_count"` + DefaultBranch string `json:"default_branch"` + Namespace string `json:"namespace"` + NameWithNamespace string `json:"name_with_namespace"` + PathWithNamespace string `json:"path_with_namespace"` +} + +type EventEnterprise struct { + Name string `json:"name"` + URL string `json:"url"` +} + +type EventUser struct { + Login string `json:"login"` + AvatarURL string `json:"avatar_url"` + HTMLURL string `json:"html_url"` + Type string `json:"type"` + SiteAdmin bool `json:"site_admin"` + ID int `json:"id"` + Name string `json:"name"` + Email string `json:"email"` + Username string `json:"username"` + UserName string `json:"user_name"` + URL string `json:"url"` +} diff --git a/pkg/tool/gitee/organizations.go b/pkg/tool/gitee/organizations.go new file mode 100644 index 0000000000000000000000000000000000000000..a9f88be9ec1dc4cfedfa7b50de3e609538c5dc45 --- /dev/null +++ b/pkg/tool/gitee/organizations.go @@ -0,0 +1,34 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gitee + +import ( + "context" + + "gitee.com/openeuler/go-gitee/gitee" + "github.com/antihax/optional" +) + +func (c *Client) ListOrganizationsForAuthenticatedUser(ctx context.Context) ([]gitee.Group, error) { + resp, _, err := c.OrganizationsApi.GetV5UserOrgs(ctx, &gitee.GetV5UserOrgsOpts{ + PerPage: optional.NewInt32(100), + }) + if err != nil { + return nil, err + } + return resp, nil +} diff --git a/pkg/tool/gitee/pull_requests.go b/pkg/tool/gitee/pull_requests.go new file mode 100644 index 0000000000000000000000000000000000000000..16da107fd4ca1c09e1913bab8df5094128def357 --- /dev/null +++ b/pkg/tool/gitee/pull_requests.go @@ -0,0 +1,58 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gitee + +import ( + "context" + + "gitee.com/openeuler/go-gitee/gitee" +) + +func (c *Client) GetPullRequest(ctx context.Context, owner string, repo string, number int) (gitee.PullRequest, error) { + pr, _, err := c.PullRequestsApi.GetV5ReposOwnerRepoPullsNumber(ctx, owner, repo, int32(number), nil) + if err != nil { + return gitee.PullRequest{}, err + } + + return pr, err +} + +func (c *Client) ListPullRequests(ctx context.Context, owner string, repo string, opts *gitee.GetV5ReposOwnerRepoPullsOpts) ([]gitee.PullRequest, error) { + prs, _, err := c.PullRequestsApi.GetV5ReposOwnerRepoPulls(ctx, owner, repo, opts) + if err != nil { + return nil, err + } + + return prs, err +} + +func (c *Client) ListCommits(ctx context.Context, owner string, repo string, number int, opts *gitee.GetV5ReposOwnerRepoPullsNumberCommitsOpts) ([]gitee.PullRequestCommits, error) { + cs, _, err := c.PullRequestsApi.GetV5ReposOwnerRepoPullsNumberCommits(ctx, owner, repo, int32(number), opts) + if err != nil { + return nil, err + } + + return cs, err +} + +func (c *Client) ListFiles(ctx context.Context, owner string, repo string, number int, opts *gitee.GetV5ReposOwnerRepoPullsNumberFilesOpts) ([]gitee.PullRequestFiles, error) { + is, _, err := c.PullRequestsApi.GetV5ReposOwnerRepoPullsNumberFiles(ctx, owner, repo, int32(number), opts) + if err != nil { + return nil, err + } + return is, err +} diff --git a/pkg/tool/gitee/repositories.go b/pkg/tool/gitee/repositories.go new file mode 100644 index 0000000000000000000000000000000000000000..124223aef710dbc2a0ac37442ed980ca5a3d9924 --- /dev/null +++ b/pkg/tool/gitee/repositories.go @@ -0,0 +1,329 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gitee + +import ( + "context" + "fmt" + "strconv" + "time" + + "gitee.com/openeuler/go-gitee/gitee" + "github.com/antihax/optional" + + "github.com/koderover/zadig/pkg/tool/git" + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +const ( + GiteeHOSTURL = "https://gitee.com/api" +) + +type Project struct { + ID int `json:"id"` + Name string `json:"name"` + DefaultBranch string `json:"default_branch,omitempty"` +} + +func (c *Client) ListRepositoriesForAuthenticatedUser(accessToken, keyword string, page, perPage int) ([]Project, error) { + httpClient := httpclient.New( + httpclient.SetHostURL(GiteeHOSTURL), + ) + // api reference: https://gitee.com/api/v5/swagger#/getV5UserRepos + url := "/v5/user/repos" + queryParams := make(map[string]string) + queryParams["access_token"] = accessToken + //queryParams["visibility"] = "all" + //queryParams["affiliation"] = "owner" + queryParams["type"] = "personal" + queryParams["q"] = keyword + queryParams["page"] = strconv.Itoa(page) + queryParams["per_page"] = strconv.Itoa(perPage) + + var projects []Project + _, err := httpClient.Get(url, httpclient.SetQueryParams(queryParams), httpclient.SetResult(&projects)) + if err != nil { + return nil, err + } + + return projects, nil +} + +func (c *Client) ListRepositoriesForOrg(accessToken, org string, page, perPage int) ([]Project, error) { + httpClient := httpclient.New( + httpclient.SetHostURL(GiteeHOSTURL), + ) + // api reference: https://gitee.com/api/v5/swagger#/getV5OrgsOrgRepos + url := fmt.Sprintf("/v5/orgs/%s/repos", org) + queryParams := make(map[string]string) + queryParams["access_token"] = accessToken + queryParams["type"] = "all" + queryParams["page"] = strconv.Itoa(page) + queryParams["per_page"] = strconv.Itoa(perPage) + + var projects []Project + _, err := httpClient.Get(url, httpclient.SetQueryParams(queryParams), httpclient.SetResult(&projects)) + if err != nil { + return nil, err + } + + return projects, nil +} + +func (c *Client) ListHooks(ctx context.Context, owner, repo string, opts *gitee.GetV5ReposOwnerRepoHooksOpts) ([]gitee.Hook, error) { + hs, _, err := c.WebhooksApi.GetV5ReposOwnerRepoHooks(ctx, owner, repo, opts) + if err != nil { + return nil, err + } + return hs, nil +} + +func (c *Client) DeleteHook(ctx context.Context, owner, repo string, id int64) error { + _, err := c.WebhooksApi.DeleteV5ReposOwnerRepoHooksId(ctx, owner, repo, int32(id), nil) + if err != nil { + return err + } + return nil +} + +type Hook struct { + ID int `json:"id"` + URL string `json:"url"` + CreatedAt time.Time `json:"created_at"` + Password string `json:"password"` + ProjectID int `json:"project_id"` + Result string `json:"result"` + ResultCode int `json:"result_code"` + PushEvents bool `json:"push_events"` + TagPushEvents bool `json:"tag_push_events"` + IssuesEvents bool `json:"issues_events"` + NoteEvents bool `json:"note_events"` + MergeRequestsEvents bool `json:"merge_requests_events"` +} + +func (c *Client) CreateHook(accessToken, owner, repo string, hook *git.Hook) (*Hook, error) { + httpClient := httpclient.New( + httpclient.SetHostURL(GiteeHOSTURL), + ) + url := fmt.Sprintf("/v5/repos/%s/%s/hooks", owner, repo) + var hookInfo *Hook + _, err := httpClient.Post(url, httpclient.SetBody(struct { + AccessToken string `json:"access_token"` + URL string `json:"url"` + Password string `json:"password"` + PushEvents string `json:"push_events"` + TagPushEvents string `json:"tag_push_events"` + MergeRequestsEvents string `json:"merge_requests_events"` + }{accessToken, hook.URL, hook.Secret, "true", "true", "true"}), httpclient.SetResult(&hookInfo)) + if err != nil { + return nil, err + } + return hookInfo, nil +} + +func (c *Client) UpdateHook(ctx context.Context, owner, repo string, id int64, hook *git.Hook) (gitee.Hook, error) { + resp, _, err := c.WebhooksApi.PatchV5ReposOwnerRepoHooksId(ctx, owner, repo, int32(id), hook.URL, &gitee.PatchV5ReposOwnerRepoHooksIdOpts{ + Password: optional.NewString(hook.Secret), + PushEvents: optional.NewBool(true), + TagPushEvents: optional.NewBool(true), + MergeRequestsEvents: optional.NewBool(true), + }) + if err != nil { + return gitee.Hook{}, err + } + + return resp, nil +} + +func (c *Client) GetContents(ctx context.Context, owner, repo, sha string) (gitee.Blob, error) { + fileContent, _, err := c.GitDataApi.GetV5ReposOwnerRepoGitBlobsSha(ctx, owner, repo, sha, &gitee.GetV5ReposOwnerRepoGitBlobsShaOpts{}) + if err != nil { + return fileContent, err + } + return fileContent, nil +} + +// "Recursive" Assign a value of 1 to get the directory recursively +// sha Can be a branch name (such as master), Commit, or the SHA value of the directory Tree +func (c *Client) GetTrees(ctx context.Context, owner, repo, sha string, level int) (gitee.Tree, error) { + tree, _, err := c.GitDataApi.GetV5ReposOwnerRepoGitTreesSha(ctx, owner, repo, sha, &gitee.GetV5ReposOwnerRepoGitTreesShaOpts{ + Recursive: optional.NewInt32(int32(level)), + }) + if err != nil { + return gitee.Tree{}, err + } + return tree, nil +} + +type RepoCommit struct { + URL string `json:"url"` + Sha string `json:"sha"` + Commit RepoCommitInnerCommit `json:"commit"` +} + +type RepoCommitInnerCommit struct { + Author RepoCommitAuthor `json:"author"` + Committer RepoCommitAuthor `json:"committer"` + Message string `json:"message"` +} + +type RepoCommitAuthor struct { + Name string `json:"name"` + Date time.Time `json:"date"` + Email string `json:"email"` +} + +func (c *Client) GetSingleCommitOfProject(ctx context.Context, accessToken, owner, repo, commitSha string) (*RepoCommit, error) { + httpClient := httpclient.New( + httpclient.SetHostURL(GiteeHOSTURL), + ) + url := fmt.Sprintf("/v5/repos/%s/%s/commits/%s", owner, repo, commitSha) + + var commit *RepoCommit + _, err := httpClient.Get(url, httpclient.SetQueryParam("access_token", accessToken), httpclient.SetResult(&commit)) + if err != nil { + return nil, err + } + return commit, nil +} + +type AccessToken struct { + AccessToken string `json:"access_token"` + TokenType string `json:"token_type"` + ExpiresIn int `json:"expires_in"` + RefreshToken string `json:"refresh_token"` + Scope string `json:"scope"` + CreatedAt int `json:"created_at"` +} + +func RefreshAccessToken(refreshToken string) (*AccessToken, error) { + httpClient := httpclient.New( + httpclient.SetHostURL("https://gitee.com"), + ) + url := "/oauth/token" + queryParams := make(map[string]string) + queryParams["grant_type"] = "refresh_token" + queryParams["refresh_token"] = refreshToken + + var accessToken *AccessToken + _, err := httpClient.Post(url, httpclient.SetQueryParams(queryParams), httpclient.SetResult(&accessToken)) + if err != nil { + return nil, err + } + + return accessToken, nil +} + +type Compare struct { + Commits []CompareCommit `json:"commits"` + Files []CompareFiles `json:"files"` +} + +type CompareCommit struct { + URL string `json:"url"` + Sha string `json:"sha"` + HTMLURL string `json:"html_url"` + CommentsURL string `json:"comments_url"` + Commit CompareInnerCommit `json:"commit"` + Author CompareAuthor `json:"author"` + Committer CompareCommitter `json:"committer"` + Parents []CompareParents `json:"parents"` +} + +type CompareParents struct { + Sha string `json:"sha"` + URL string `json:"url"` +} + +type CompareInnerCommit struct { + Author RepoCommitAuthor `json:"author"` + Committer RepoCommitAuthor `json:"committer"` + Message string `json:"message"` + Tree CompareTree `json:"tree"` +} + +type CompareTree struct { + Sha string `json:"sha"` + URL string `json:"url"` +} + +type CompareFiles struct { + Sha string `json:"sha"` + Filename string `json:"filename"` + Status string `json:"status"` + Additions int `json:"additions"` + Deletions int `json:"deletions"` + Changes int `json:"changes"` + BlobURL string `json:"blob_url"` + RawURL string `json:"raw_url"` + ContentURL string `json:"content_url"` + Patch string `json:"patch"` +} + +type CompareCommitter struct { + ID int `json:"id"` + Login string `json:"login"` + Name string `json:"name"` + AvatarURL string `json:"avatar_url"` + URL string `json:"url"` + HTMLURL string `json:"html_url"` + Remark string `json:"remark"` + FollowersURL string `json:"followers_url"` + FollowingURL string `json:"following_url"` + GistsURL string `json:"gists_url"` + StarredURL string `json:"starred_url"` + SubscriptionsURL string `json:"subscriptions_url"` + OrganizationsURL string `json:"organizations_url"` + ReposURL string `json:"repos_url"` + EventsURL string `json:"events_url"` + ReceivedEventsURL string `json:"received_events_url"` + Type string `json:"type"` +} + +type CompareAuthor struct { + ID int `json:"id"` + Login string `json:"login"` + Name string `json:"name"` + AvatarURL string `json:"avatar_url"` + URL string `json:"url"` + HTMLURL string `json:"html_url"` + Remark string `json:"remark"` + FollowersURL string `json:"followers_url"` + FollowingURL string `json:"following_url"` + GistsURL string `json:"gists_url"` + StarredURL string `json:"starred_url"` + SubscriptionsURL string `json:"subscriptions_url"` + OrganizationsURL string `json:"organizations_url"` + ReposURL string `json:"repos_url"` + EventsURL string `json:"events_url"` + ReceivedEventsURL string `json:"received_events_url"` + Type string `json:"type"` +} + +func (c *Client) GetReposOwnerRepoCompareBaseHead(accessToken, owner string, repo string, base string, head string) (*Compare, error) { + httpClient := httpclient.New( + httpclient.SetHostURL(GiteeHOSTURL), + ) + url := fmt.Sprintf("/v5/repos/%s/%s/compare/%s...%s", owner, repo, base, head) + + var compare *Compare + _, err := httpClient.Get(url, httpclient.SetQueryParam("access_token", accessToken), httpclient.SetResult(&compare)) + if err != nil { + return nil, err + } + return compare, nil +} diff --git a/pkg/tool/gitee/tag.go b/pkg/tool/gitee/tag.go new file mode 100644 index 0000000000000000000000000000000000000000..45117317d0567371b622e5761b60f285a6048d47 --- /dev/null +++ b/pkg/tool/gitee/tag.go @@ -0,0 +1,49 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gitee + +import ( + "context" + "fmt" + "time" + + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +type Tag struct { + Name string `json:"name"` + Message string `json:"message"` + Commit Commit `json:"commit"` +} + +type Commit struct { + Sha string `json:"sha"` + Date time.Time `json:"date"` +} + +func (c *Client) ListTags(ctx context.Context, accessToken, owner string, repo string) ([]Tag, error) { + httpClient := httpclient.New( + httpclient.SetHostURL(GiteeHOSTURL), + ) + url := fmt.Sprintf("/v5/repos/%s/%s/tags", owner, repo) + var tags []Tag + _, err := httpClient.Get(url, httpclient.SetQueryParam("access_token", accessToken), httpclient.SetResult(&tags)) + if err != nil { + return nil, err + } + return tags, nil +} diff --git a/pkg/tool/gitee/users.go b/pkg/tool/gitee/users.go new file mode 100644 index 0000000000000000000000000000000000000000..f46fa999a2e4c467a3d47f945cedca4daf715acb --- /dev/null +++ b/pkg/tool/gitee/users.go @@ -0,0 +1,32 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gitee + +import ( + "context" + + "gitee.com/openeuler/go-gitee/gitee" +) + +func (c *Client) GetAuthenticatedUser(ctx context.Context) (gitee.User, error) { + ur, _, err := c.UsersApi.GetV5User(ctx, &gitee.GetV5UserOpts{}) + if err != nil { + return gitee.User{}, err + } + + return ur, err +} diff --git a/pkg/tool/gorm/client.go b/pkg/tool/gorm/client.go new file mode 100644 index 0000000000000000000000000000000000000000..93b0f5dec946dcc67fb8de86b3f93ef838bae2d6 --- /dev/null +++ b/pkg/tool/gorm/client.go @@ -0,0 +1,61 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gorm + +import ( + "fmt" + + "gorm.io/driver/mysql" + "gorm.io/gorm" +) + +var connections = make(map[string]*gorm.DB) + +func getConnection(db string) *gorm.DB { + return connections[db] +} + +func Open(username, password, host, db string) error { + conn, err := openDB(username, password, host, db) + if err != nil { + return err + } + + connections[db] = conn + + return nil +} + +func DB(db string) *gorm.DB { + return getConnection(db) +} + +func Close() { + //for _, conn := range connections { + // conn.Close() + //} +} + +func openDB(username, password, host, db string) (*gorm.DB, error) { + + // refer https://github.com/go-sql-driver/mysql#dsn-data-source-name for details + dsn := fmt.Sprintf( + "%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", + username, password, host, db, + ) + return gorm.Open(mysql.Open(dsn), &gorm.Config{}) +} diff --git a/pkg/tool/helmclient/helmclient.go b/pkg/tool/helmclient/helmclient.go index f786d63c001066cd6bbf784d7b25ef806f4455ce..1cbf8a3b76655b4250050ec3b4a36d852a632be0 100644 --- a/pkg/tool/helmclient/helmclient.go +++ b/pkg/tool/helmclient/helmclient.go @@ -17,50 +17,172 @@ limitations under the License. package helmclient import ( + "bytes" + "context" "encoding/json" + "errors" "fmt" + "io/ioutil" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "reflect" "strings" + "sync" + cm "github.com/chartmuseum/helm-push/pkg/chartmuseum" hc "github.com/mittwald/go-helm-client" + "helm.sh/helm/v3/pkg/action" + "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chart/loader" + "helm.sh/helm/v3/pkg/cli" + "helm.sh/helm/v3/pkg/downloader" + "helm.sh/helm/v3/pkg/plugin" + "helm.sh/helm/v3/pkg/release" + "helm.sh/helm/v3/pkg/releaseutil" + "helm.sh/helm/v3/pkg/repo" + "helm.sh/helm/v3/pkg/storage" + "helm.sh/helm/v3/pkg/storage/driver" "helm.sh/helm/v3/pkg/strvals" + v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/yaml" + "github.com/koderover/zadig/pkg/microservice/aslan/config" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" + "github.com/koderover/zadig/pkg/tool/kube/updater" "github.com/koderover/zadig/pkg/tool/log" yamlutil "github.com/koderover/zadig/pkg/util/yaml" ) +const ( + HelmPluginsDirectory = "/app/.helm/helmplugin" +) + +var repoInfo *repo.File +var generalSettings *cli.EnvSettings + +// enable support of oci registry +func init() { + os.Setenv("HELM_EXPERIMENTAL_OCI", "1") + os.Setenv("HELM_PLUGINS", HelmPluginsDirectory) + repoInfo = &repo.File{} + generalSettings = cli.New() + generalSettings.PluginsDirectory = HelmPluginsDirectory +} + +type HelmClient struct { + *hc.HelmClient + kubeClient client.Client + Namespace string + lock *sync.Mutex + RestConfig *rest.Config +} + +// NewClient returns a new Helm client with no construct parameters +// used to update helm repo data and download index.yaml or helm charts +func NewClient() (*HelmClient, error) { + hcClient, err := hc.New(&hc.Options{ + RepositoryConfig: generalSettings.RepositoryConfig, + RepositoryCache: generalSettings.RepositoryCache, + }) + + if err != nil { + return nil, err + } + helmClient := hcClient.(*hc.HelmClient) + helmClient.Settings = generalSettings + return &HelmClient{ + helmClient, + nil, + "", + &sync.Mutex{}, + nil, + }, nil +} + +// NewClientFromNamespace returns a new Helm client constructed with the provided clusterID and namespace +// a kubeClient will be initialized to support necessary k8s operations when install/upgrade helm charts +func NewClientFromNamespace(clusterID, namespace string) (*HelmClient, error) { + restConfig, err := kubeclient.GetRESTConfig(config.HubServerAddress(), clusterID) + if err != nil { + return nil, err + } + + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), clusterID) + if err != nil { + return nil, err + } + + hcClient, err := hc.NewClientFromRestConf(&hc.RestConfClientOptions{ + Options: &hc.Options{ + Namespace: namespace, + DebugLog: log.Debugf, + }, + RestConfig: restConfig, + }) + if err != nil { + return nil, err + } + + helmClient := hcClient.(*hc.HelmClient) + return &HelmClient{ + helmClient, + kubeClient, + namespace, + &sync.Mutex{}, + restConfig, + }, nil +} + // NewClientFromRestConf returns a new Helm client constructed with the provided REST config options -func NewClientFromRestConf(restConfig *rest.Config, ns string) (hc.Client, error) { - return hc.NewClientFromRestConf(&hc.RestConfClientOptions{ +// used to list/uninstall helm release +func NewClientFromRestConf(restConfig *rest.Config, ns string) (*HelmClient, error) { + hcClient, err := hc.NewClientFromRestConf(&hc.RestConfClientOptions{ Options: &hc.Options{ Namespace: ns, DebugLog: log.Debugf, }, RestConfig: restConfig, }) + if err != nil { + return nil, err + } + + helmClient := hcClient.(*hc.HelmClient) + return &HelmClient{ + helmClient, + nil, + ns, + &sync.Mutex{}, + restConfig, + }, nil } type KV struct { - Key string `json:"key"` - Value string `json:"value"` + Key string `json:"key"` + Value interface{} `json:"value"` } -// MergeOverrideValues merge values.yaml and override values -// overrideYaml used for -f option +// MergeOverrideValues merge override yaml and override kvs +// defaultValues overrideYaml used for -f option // overrideValues used for --set option -func MergeOverrideValues(valuesYaml, overrideYaml, overrideValues string) (string, error) { - - if overrideYaml == "" && overrideValues == "" { - return valuesYaml, nil - } +func MergeOverrideValues(valuesYaml, defaultValues, overrideYaml, overrideValues string) (string, error) { - // merge files for -f option - values, err := yamlutil.MergeAndUnmarshal([][]byte{[]byte(valuesYaml), []byte(overrideYaml)}) + // merge files for helm -f option + // precedence from low to high: valuesYaml defaultValues overrideYaml + valuesMap, err := yamlutil.MergeAndUnmarshal([][]byte{[]byte(valuesYaml), []byte(defaultValues), []byte(overrideYaml)}) if err != nil { return "", err } + // merge kv values for helm --set option if overrideValues != "" { kvList := make([]*KV, 0) err = json.Unmarshal([]byte(overrideValues), &kvList) @@ -69,18 +191,553 @@ func MergeOverrideValues(valuesYaml, overrideYaml, overrideValues string) (strin } kvStr := make([]string, 0) for _, kv := range kvList { - kvStr = append(kvStr, fmt.Sprintf("%s=%s", kv.Key, kv.Value)) + kvStr = append(kvStr, fmt.Sprintf("%s=%v", kv.Key, kv.Value)) } // override values for --set option - err = strvals.ParseInto(strings.Join(kvStr, ","), values) + err = strvals.ParseInto(strings.Join(kvStr, ","), valuesMap) if err != nil { return "", err } } - bs, err := yaml.Marshal(values) + bs, err := yaml.Marshal(valuesMap) if err != nil { return "", err } return string(bs), nil } + +// upgradeCRDs upgrades the CRDs of the provided chart. +func (hClient *HelmClient) upgradeCRDs(ctx context.Context, chartInstance *chart.Chart) error { + cfg, err := hClient.ActionConfig.RESTClientGetter.ToRESTConfig() + if err != nil { + return err + } + + k8sClient, err := clientset.NewForConfig(cfg) + if err != nil { + return err + } + + for _, crd := range chartInstance.CRDObjects() { + if err := hClient.upgradeCRD(ctx, k8sClient, crd); err != nil { + return err + } + hClient.DebugLog("CRD %s upgraded successfully for chart: %s", crd.Name, chartInstance.Metadata.Name) + } + + return nil +} + +// upgradeCRDV1Beta1 upgrades a CRD of the v1 API version using the provided k8s client and CRD yaml. +func (hClient *HelmClient) upgradeCRDV1(ctx context.Context, cl *clientset.Clientset, rawCRD []byte) error { + var crdObj v1.CustomResourceDefinition + if err := yaml.Unmarshal(rawCRD, &crdObj); err != nil { + return err + } + + existingCRDObj, err := cl.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, crdObj.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + // Check to ensure that no previously existing API version is deleted through the upgrade. + if len(existingCRDObj.Spec.Versions) > len(crdObj.Spec.Versions) { + hClient.DebugLog("WARNING: new version of CRD %q would remove an existing API version, skipping upgrade", crdObj.Name) + return nil + } + + // Check that the storage version does not change through the update. + oldStorageVersion := v1.CustomResourceDefinitionVersion{} + + for _, oldVersion := range existingCRDObj.Spec.Versions { + if oldVersion.Storage { + oldStorageVersion = oldVersion + } + } + + i := 0 + + for _, newVersion := range crdObj.Spec.Versions { + if newVersion.Storage { + i++ + if newVersion.Name != oldStorageVersion.Name { + return fmt.Errorf("ERROR: storage version of CRD %q changed, aborting upgrade", crdObj.Name) + } + } + if i > 1 { + return fmt.Errorf("ERROR: more than one storage version set on CRD %q, aborting upgrade", crdObj.Name) + } + } + + if reflect.DeepEqual(existingCRDObj.Spec.Versions, crdObj.Spec.Versions) { + hClient.DebugLog("INFO: new version of CRD %q contains no changes, skipping upgrade", crdObj.Name) + return nil + } + + crdObj.ResourceVersion = existingCRDObj.ResourceVersion + if _, err := cl.ApiextensionsV1().CustomResourceDefinitions().Update(ctx, &crdObj, metav1.UpdateOptions{DryRun: []string{"All"}}); err != nil { + return err + } + hClient.DebugLog("upgrade ran successful for CRD (dry run): %s", crdObj.Name) + + if _, err := cl.ApiextensionsV1().CustomResourceDefinitions().Update(ctx, &crdObj, metav1.UpdateOptions{}); err != nil { + return err + } + hClient.DebugLog("upgrade ran successful for CRD: %s", crdObj.Name) + + return nil +} + +// upgradeCRDV1Beta1 upgrades a CRD of the v1beta1 API version using the provided k8s client and CRD yaml. +func (hClient *HelmClient) upgradeCRDV1Beta1(ctx context.Context, cl *clientset.Clientset, rawCRD []byte) error { + var crdObj v1beta1.CustomResourceDefinition + if err := yaml.Unmarshal(rawCRD, &crdObj); err != nil { + return err + } + existingCRDObj, err := cl.ApiextensionsV1beta1().CustomResourceDefinitions().Get(ctx, crdObj.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + // Check that the storage version does not change through the update. + oldStorageVersion := v1beta1.CustomResourceDefinitionVersion{} + + for _, oldVersion := range existingCRDObj.Spec.Versions { + if oldVersion.Storage { + oldStorageVersion = oldVersion + } + } + + i := 0 + + for _, newVersion := range crdObj.Spec.Versions { + if newVersion.Storage { + i++ + if newVersion.Name != oldStorageVersion.Name { + return fmt.Errorf("ERROR: storage version of CRD %q changed, aborting upgrade", crdObj.Name) + } + } + if i > 1 { + return fmt.Errorf("ERROR: more than one storage version set on CRD %q, aborting upgrade", crdObj.Name) + } + } + + if reflect.DeepEqual(existingCRDObj.Spec.Versions, crdObj.Spec.Versions) { + hClient.DebugLog("INFO: new version of CRD %q contains no changes, skipping upgrade", crdObj.Name) + return nil + } + + crdObj.ResourceVersion = existingCRDObj.ResourceVersion + if _, err := cl.ApiextensionsV1beta1().CustomResourceDefinitions().Update(ctx, &crdObj, metav1.UpdateOptions{DryRun: []string{"All"}}); err != nil { + return err + } + hClient.DebugLog("upgrade ran successful for CRD (dry run): %s", crdObj.Name) + + if _, err = cl.ApiextensionsV1beta1().CustomResourceDefinitions().Update(ctx, &crdObj, metav1.UpdateOptions{}); err != nil { + return err + } + hClient.DebugLog("upgrade ran successful for CRD: %s", crdObj.Name) + + return nil +} + +// upgradeCRD upgrades the CRD 'crd' using the provided k8s client. +func (hClient *HelmClient) upgradeCRD(ctx context.Context, k8sClient *clientset.Clientset, crd chart.CRD) error { + var typeMeta metav1.TypeMeta + err := yaml.Unmarshal(crd.File.Data, &typeMeta) + if err != nil { + return err + } + + switch typeMeta.APIVersion { + case "apiextensions.k8s.io/v1beta1": + return hClient.upgradeCRDV1Beta1(ctx, k8sClient, crd.File.Data) + case "apiextensions.k8s.io/v1": + return hClient.upgradeCRDV1(ctx, k8sClient, crd.File.Data) + default: + return fmt.Errorf("WARNING: failed to upgrade CRD %q: unsupported api-version %q", crd.Name, typeMeta.APIVersion) + } +} + +// check weather to install or upgrade chart by current status +// return error if neither install nor upgrade action is legal +func (hClient *HelmClient) isInstallOperation(spec *hc.ChartSpec) (bool, error) { + historyReleaseCount := 10 + if spec.MaxHistory > 0 { + historyReleaseCount = spec.MaxHistory + } + // find history of particular release + releases, err := hClient.ListReleaseHistory(spec.ReleaseName, historyReleaseCount) + if err != nil && err != driver.ErrReleaseNotFound { + return false, err + } + // release not found, install operation + if len(releases) == 0 { + return true, nil + } + + releaseutil.Reverse(releases, releaseutil.SortByRevision) + lastRelease := releases[0] + + // pending status + if lastRelease.Info.Status.IsPending() { + return false, errors.New("another operation (install/upgrade/rollback) is in progress, please try later") + } + + // find deployed revision with status deployed from history, would be upgrade operation + for _, rel := range releases { + if rel.Info.Status == release.StatusDeployed { + return false, nil + } + } + + // release with failed/superseded status: legal upgrade operation + if lastRelease.Info.Status == release.StatusFailed || lastRelease.Info.Status == release.StatusSuperseded { + return false, hClient.ensureUpgrade(historyReleaseCount, spec.ReleaseName, releases) + } + + // if replace set to true, install will be a legal operation + if st := lastRelease.Info.Status; spec.Replace && (st == release.StatusUninstalled || st == release.StatusFailed) { + return true, nil + } + + return false, fmt.Errorf("can't install or upgrade chart with status: %s", lastRelease.Info.Status) +} + +// ensure new release revision can be saved +func (hClient *HelmClient) ensureUpgrade(maxHistoryCount int, releaseName string, releases []*release.Release) error { + if maxHistoryCount <= 0 || len(releases) < maxHistoryCount { + return nil + } + if hClient.kubeClient == nil { + return errors.New("kubeClient is nil") + } + secretName := fmt.Sprintf("%s.%s.v%d", storage.HelmStorageType, releaseName, releases[len(releases)-1].Version) + return updater.DeleteSecretWithName(hClient.Namespace, secretName, hClient.kubeClient) +} + +// getChart returns a chart matching the provided chart name and options. +func (hClient *HelmClient) getChart(chartName string, chartPathOptions *action.ChartPathOptions) (*chart.Chart, string, error) { + chartPath, err := chartPathOptions.LocateChart(chartName, hClient.HelmClient.Settings) + if err != nil { + return nil, "", err + } + + helmChart, err := loader.Load(chartPath) + if err != nil { + return nil, "", err + } + + if helmChart.Metadata.Deprecated { + hClient.HelmClient.DebugLog("WARNING: This chart (%q) is deprecated", helmChart.Metadata.Name) + } + + return helmChart, chartPath, err +} + +func (hClient *HelmClient) installChart(ctx context.Context, spec *hc.ChartSpec) (*release.Release, error) { + c := hClient.HelmClient + install := action.NewInstall(c.ActionConfig) + mergeInstallOptions(spec, install) + + if install.Version == "" { + install.Version = ">0.0.0-0" + } + + helmChart, chartPath, err := hClient.getChart(spec.ChartName, &install.ChartPathOptions) + if err != nil { + return nil, err + } + + if helmChart.Metadata.Type != "" && helmChart.Metadata.Type != "application" { + return nil, fmt.Errorf( + "chart %q has an unsupported type and is not installable: %q", + helmChart.Metadata.Name, + helmChart.Metadata.Type, + ) + } + + if req := helmChart.Metadata.Dependencies; req != nil { + if err := action.CheckDependencies(helmChart, req); err != nil { + if !install.DependencyUpdate { + return nil, err + } + man := &downloader.Manager{ + ChartPath: chartPath, + Keyring: install.ChartPathOptions.Keyring, + SkipUpdate: false, + Getters: c.Providers, + RepositoryConfig: generalSettings.RepositoryConfig, + RepositoryCache: generalSettings.RepositoryCache, + } + if err := man.Update(); err != nil { + return nil, err + } + } + } + + values, err := spec.GetValuesMap() + if err != nil { + return nil, err + } + + rel, err := install.RunWithContext(ctx, helmChart, values) + if err != nil { + return rel, err + } + + c.DebugLog("release installed successfully: %s/%s-%s", rel.Name, rel.Chart.Metadata.Name, rel.Chart.Metadata.Version) + + return rel, nil +} + +func (hClient *HelmClient) upgradeChart(ctx context.Context, spec *hc.ChartSpec) (*release.Release, error) { + c := hClient.HelmClient + upgrade := action.NewUpgrade(c.ActionConfig) + mergeUpgradeOptions(spec, upgrade) + + if upgrade.Version == "" { + upgrade.Version = ">0.0.0-0" + } + + helmChart, _, err := hClient.getChart(spec.ChartName, &upgrade.ChartPathOptions) + if err != nil { + return nil, err + } + + if req := helmChart.Metadata.Dependencies; req != nil { + if err := action.CheckDependencies(helmChart, req); err != nil { + return nil, err + } + } + + values, err := spec.GetValuesMap() + if err != nil { + return nil, err + } + + if !spec.SkipCRDs && spec.UpgradeCRDs { + c.DebugLog("upgrading crds") + err = hClient.upgradeCRDs(ctx, helmChart) + if err != nil { + return nil, err + } + } + + rel, err := upgrade.RunWithContext(ctx, spec.ReleaseName, helmChart, values) + if err != nil { + return rel, err + } + c.DebugLog("release upgraded successfully: %s/%s-%s", rel.Name, rel.Chart.Metadata.Name, rel.Chart.Metadata.Version) + return rel, nil +} + +// InstallOrUpgradeChart install or upgrade helm chart, use the same rule with helm to determine weather to install or upgrade +func (hClient *HelmClient) InstallOrUpgradeChart(ctx context.Context, spec *hc.ChartSpec) (*release.Release, error) { + install, err := hClient.isInstallOperation(spec) + if err != nil { + return nil, err + } + + if install { + return hClient.installChart(ctx, spec) + } else { + return hClient.upgradeChart(ctx, spec) + } +} + +// UpdateChartRepo works like executing `helm repo update` +// environment `HELM_REPO_USERNAME` and `HELM_REPO_PASSWORD` are only required for ali acr repos +func (hClient *HelmClient) UpdateChartRepo(repoEntry *repo.Entry) (string, error) { + chartRepo, err := repo.NewChartRepository(repoEntry, hClient.Providers) + if err != nil { + return "", err + } + chartRepo.CachePath = hClient.Settings.RepositoryCache + + repoUrl, err := url.Parse(repoEntry.URL) + if err != nil { + return "", fmt.Errorf("failed to parse repo url: %s, err: %w", repoEntry.URL, err) + } + if repoUrl.Scheme == "acr" { + // export envionment-variables for ali acr chart repo + os.Setenv("HELM_REPO_USERNAME", repoEntry.Username) + os.Setenv("HELM_REPO_PASSWORD", repoEntry.Password) + } + + // update repo info + repoInfo.Update(repoEntry) + + // download index.yaml + indexFilePath, err := chartRepo.DownloadIndexFile() + if err != nil { + return "", err + } + + err = repoInfo.WriteFile(hClient.Settings.RepositoryConfig, 0o644) + if err != nil { + return "", err + } + return indexFilePath, err +} + +// FetchIndexYaml fetch index.yaml from remote chart repo +// `helm repo add` and `helm repo update` will be executed +func (hClient *HelmClient) FetchIndexYaml(repoEntry *repo.Entry) (*repo.IndexFile, error) { + hClient.lock.Lock() + defer hClient.lock.Unlock() + indexFilePath, err := hClient.UpdateChartRepo(repoEntry) + if err != nil { + return nil, err + } + // Read the index file for the repository to get chart information and return chart URL + repoIndex, err := repo.LoadIndexFile(indexFilePath) + return repoIndex, err +} + +// DownloadChart works like executing `helm pull repoName/chartName --version=version' +// since pulling from OCI Registry is still considered as an EXPERIMENTAL feature +// we DO NOT support pulling charts by pulling OCI Artifacts from OCI Registry +// NOTE consider using os.execCommand('helm pull') to reduce code complexity of offering compatibility since third-party plugins CANNOT be used as SDK +func (hClient *HelmClient) DownloadChart(repoEntry *repo.Entry, chartRef string, chartVersion string, destDir string, unTar bool) error { + hClient.lock.Lock() + defer hClient.lock.Unlock() + _, err := hClient.UpdateChartRepo(repoEntry) + if err != nil { + return nil + } + pull := action.NewPull() + pull.Password = repoEntry.Username + pull.Username = repoEntry.Password + pull.Version = chartVersion + pull.Settings = generalSettings + pull.DestDir = destDir + pull.UntarDir = destDir + pull.Untar = unTar + _, err = pull.Run(chartRef) + return err +} + +func (hClient *HelmClient) pushAcrChart(repoEntry *repo.Entry, chartPath string) error { + base := filepath.Join(hClient.Settings.PluginsDirectory, "helm-acr") + prog := exec.Command(filepath.Join(base, "bin/helm-cm-push"), chartPath, repoEntry.Name) + plugin.SetupPluginEnv(hClient.Settings, "cm-push", base) + prog.Env = os.Environ() + buf := bytes.NewBuffer(nil) + prog.Stdout = buf + prog.Stderr = buf + if err := prog.Run(); err != nil { + if eErr, ok := err.(*exec.ExitError); ok { + return fmt.Errorf("plugin exited with error: %s %s", string(eErr.Stderr), buf.String()) + } + return fmt.Errorf("%s %s", err, buf.String()) + } + return nil +} + +func (hClient *HelmClient) pushChartMuseum(repoEntry *repo.Entry, chartPath string) error { + chartClient, err := cm.NewClient( + cm.URL(repoEntry.URL), + cm.Username(repoEntry.Username), + cm.Password(repoEntry.Password), + ) + if err != nil { + return err + } + resp, err := chartClient.UploadChartPackage(chartPath, false) + if err != nil { + return fmt.Errorf("failed to prepare pushing chart: %s, error: %w", chartPath, err) + + } + + defer resp.Body.Close() + err = handlePushResponse(resp) + if err != nil { + return fmt.Errorf("failed to push chart: %s, error: %w", chartPath, err) + } + return nil +} + +func getChartmuseumError(b []byte, code int) error { + var er struct { + Error string `json:"error"` + } + err := json.Unmarshal(b, &er) + if err != nil || er.Error == "" { + return fmt.Errorf("%d: could not properly parse response JSON: %s", code, string(b)) + } + return fmt.Errorf("chart museum errCode: %d, err: %s", code, er.Error) +} + +func handlePushResponse(resp *http.Response) error { + if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusAccepted { + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + return getChartmuseumError(b, resp.StatusCode) + } + log.Info("push chart to chart repo done") + return nil +} + +func (hClient *HelmClient) PushChart(repoEntry *repo.Entry, chartPath string) error { + hClient.lock.Lock() + defer hClient.lock.Unlock() + _, err := hClient.UpdateChartRepo(repoEntry) + if err != nil { + return nil + } + repoUrl, err := url.Parse(repoEntry.URL) + if err != nil { + return fmt.Errorf("failed to parse repo url: %s, err: %w", repoEntry.URL, err) + } + if repoUrl.Scheme == "acr" { + return hClient.pushAcrChart(repoEntry, chartPath) + } else { + return hClient.pushChartMuseum(repoEntry, chartPath) + } +} + +// NOTE: When using this method, pay attention to whether restConfig is present in the original client. +func (hClient *HelmClient) Clone() (*HelmClient, error) { + return NewClientFromRestConf(hClient.RestConfig, hClient.Namespace) +} + +// mergeInstallOptions merges values of the provided chart to helm install options used by the client. +func mergeInstallOptions(chartSpec *hc.ChartSpec, installOptions *action.Install) { + installOptions.CreateNamespace = chartSpec.CreateNamespace + installOptions.DisableHooks = chartSpec.DisableHooks + installOptions.Replace = chartSpec.Replace + installOptions.Wait = chartSpec.Wait + installOptions.DependencyUpdate = chartSpec.DependencyUpdate + installOptions.Timeout = chartSpec.Timeout + installOptions.Namespace = chartSpec.Namespace + installOptions.ReleaseName = chartSpec.ReleaseName + installOptions.Version = chartSpec.Version + installOptions.GenerateName = chartSpec.GenerateName + installOptions.NameTemplate = chartSpec.NameTemplate + installOptions.Atomic = chartSpec.Atomic + installOptions.SkipCRDs = chartSpec.SkipCRDs + installOptions.DryRun = chartSpec.DryRun + installOptions.SubNotes = chartSpec.SubNotes +} + +// mergeUpgradeOptions merges values of the provided chart to helm upgrade options used by the client. +func mergeUpgradeOptions(chartSpec *hc.ChartSpec, upgradeOptions *action.Upgrade) { + upgradeOptions.Version = chartSpec.Version + upgradeOptions.Namespace = chartSpec.Namespace + upgradeOptions.Timeout = chartSpec.Timeout + upgradeOptions.Wait = chartSpec.Wait + upgradeOptions.DisableHooks = chartSpec.DisableHooks + upgradeOptions.Force = chartSpec.Force + upgradeOptions.ResetValues = chartSpec.ResetValues + upgradeOptions.ReuseValues = chartSpec.ReuseValues + upgradeOptions.Recreate = chartSpec.Recreate + upgradeOptions.MaxHistory = chartSpec.MaxHistory + upgradeOptions.Atomic = chartSpec.Atomic + upgradeOptions.CleanupOnFail = chartSpec.CleanupOnFail + upgradeOptions.DryRun = chartSpec.DryRun + upgradeOptions.SubNotes = chartSpec.SubNotes +} diff --git a/pkg/tool/httpclient/request_options.go b/pkg/tool/httpclient/request_options.go index dde8bfd67af0f78ce81960410d7965a9bb90efef..c3b3c1bf5629adb081ca626806544e4b80f0d9ce 100644 --- a/pkg/tool/httpclient/request_options.go +++ b/pkg/tool/httpclient/request_options.go @@ -17,6 +17,7 @@ limitations under the License. package httpclient import ( + "net/http" "net/url" "github.com/go-resty/resty/v2" @@ -42,6 +43,18 @@ func SetHeader(header, value string) RequestFunc { } } +func SetHeaders(headers map[string]string) RequestFunc { + return func(r *resty.Request) { + r.SetHeaders(headers) + } +} + +func SetHeadersFromHTTPHeader(header http.Header) RequestFunc { + return func(r *resty.Request) { + r.Header = header + } +} + func SetQueryParamsFromValues(params url.Values) RequestFunc { return func(r *resty.Request) { r.SetQueryParamsFromValues(params) diff --git a/pkg/tool/ilyshin/event_parsing.go b/pkg/tool/ilyshin/event_parsing.go deleted file mode 100644 index ff06dc349ec88fd4f5b9c0c026639878ffefcbd7..0000000000000000000000000000000000000000 --- a/pkg/tool/ilyshin/event_parsing.go +++ /dev/null @@ -1,179 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ilyshin - -import ( - "encoding/json" - "fmt" - "net/http" - "time" -) - -type ProjectEvent struct { - ID int `json:"id"` - Name string `json:"name"` - Description string `json:"description"` - AvatarURL string `json:"avatar_url"` - GitSSHURL string `json:"git_ssh_url"` - GitHTTPURL string `json:"git_http_url"` - Namespace string `json:"namespace"` - PathWithNamespace string `json:"path_with_namespace"` - DefaultBranch string `json:"default_branch"` - Homepage string `json:"homepage"` - URL string `json:"url"` - SSHURL string `json:"ssh_url"` - HTTPURL string `json:"http_url"` - WebURL string `json:"web_url"` -} - -type Repository struct { - Name string `json:"name"` - Description string `json:"description"` - WebURL string `json:"web_url"` - AvatarURL string `json:"avatar_url"` - GitSSHURL string `json:"git_ssh_url"` - GitHTTPURL string `json:"git_http_url"` - Namespace string `json:"namespace"` - PathWithNamespace string `json:"path_with_namespace"` - DefaultBranch string `json:"default_branch"` - Homepage string `json:"homepage"` - URL string `json:"url"` - SSHURL string `json:"ssh_url"` - HTTPURL string `json:"http_url"` -} - -type ObjectAttributes struct { - ID int `json:"id"` - TargetBranch string `json:"target_branch"` - SourceBranch string `json:"source_branch"` - SourceProjectID int `json:"source_project_id"` - AuthorID int `json:"author_id"` - AssigneeID int `json:"assignee_id"` - AssigneeIDs []int `json:"assignee_ids"` - Title string `json:"title"` - StCommits []*Commit `json:"st_commits"` - MilestoneID int `json:"milestone_id"` - State string `json:"state"` - MergeStatus string `json:"merge_status"` - TargetProjectID int `json:"target_project_id"` - IID int `json:"iid"` - Description string `json:"description"` - Position int `json:"position"` - LockedAt string `json:"locked_at"` - UpdatedByID int `json:"updated_by_id"` - MergeError string `json:"merge_error"` - MergeWhenBuildSucceeds bool `json:"merge_when_build_succeeds"` - MergeUserID int `json:"merge_user_id"` - MergeCommitSHA string `json:"merge_commit_sha"` - DeletedAt string `json:"deleted_at"` - ApprovalsBeforeMerge string `json:"approvals_before_merge"` - RebaseCommitSHA string `json:"rebase_commit_sha"` - InProgressMergeCommitSHA string `json:"in_progress_merge_commit_sha"` - LockVersion int `json:"lock_version"` - TimeEstimate int `json:"time_estimate"` - Target *Repository `json:"target"` - LastCommit LastCommit `json:"last_commit"` - WorkInProgress bool `json:"work_in_progress"` - URL string `json:"url"` - Action string `json:"action"` - OldRev string `json:"oldrev"` -} - -type LastCommit struct { - ID string `json:"id"` - Message string `json:"message"` - Timestamp *time.Time `json:"timestamp"` - URL string `json:"url"` - Author Author `json:"author"` -} - -type MergeEvent struct { - ObjectKind string `json:"object_kind"` - Project ProjectEvent `json:"project"` - ObjectAttributes ObjectAttributes `json:"object_attributes"` -} - -type PushEvent struct { - ObjectKind string `json:"object_kind"` - Before string `json:"before"` - After string `json:"after"` - Ref string `json:"ref"` - CheckoutSHA string `json:"checkout_sha"` - UserID int `json:"user_id"` - UserName string `json:"user_name"` - UserUsername string `json:"user_username"` - UserEmail string `json:"user_email"` - UserAvatar string `json:"user_avatar"` - ProjectID int `json:"project_id"` - Project ProjectEvent `json:"project"` - Commits []Commits `json:"commits"` - TotalCommitsCount int `json:"total_commits_count"` -} - -type Commits struct { - ID string `json:"id"` - Message string `json:"message"` - Timestamp *time.Time `json:"timestamp"` - URL string `json:"url"` - Author Author `json:"author"` - Added []string `json:"added"` - Modified []string `json:"modified"` - Removed []string `json:"removed"` -} - -type Author struct { - Name string `json:"name"` - Email string `json:"email"` -} - -// EventType represents a ilyshin event type. -type EventType string - -// List of available event types. -const ( - EventTypeMergeRequest EventType = "Merge Request Hook" - EventTypePush EventType = "Push Hook" - EventTypeTagPush EventType = "Tag Push Hook" -) - -const eventTypeHeader = "X-CodeHub-Event" - -// HookEventType returns the event type for the given request. -func HookEventType(r *http.Request) EventType { - return EventType(r.Header.Get(eventTypeHeader)) -} - -func ParseHook(eventType EventType, payload []byte) (event interface{}, err error) { - return parseWebhook(eventType, payload) -} - -func parseWebhook(eventType EventType, payload []byte) (event interface{}, err error) { - switch eventType { - case EventTypeMergeRequest: - event = &MergeEvent{} - case EventTypePush: - event = &PushEvent{} - default: - return nil, fmt.Errorf("unexpected event type: %s", eventType) - } - - if err := json.Unmarshal(payload, event); err != nil { - return nil, err - } - - return event, nil -} diff --git a/pkg/tool/ilyshin/merge_request.go b/pkg/tool/ilyshin/merge_request.go deleted file mode 100644 index 0b166866224bf2cb44a5f8cc56107cad3008b357..0000000000000000000000000000000000000000 --- a/pkg/tool/ilyshin/merge_request.go +++ /dev/null @@ -1,115 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ilyshin - -import ( - "fmt" - "time" - - "go.uber.org/zap" - - "github.com/koderover/zadig/pkg/tool/httpclient" - "github.com/koderover/zadig/pkg/tool/log" -) - -type MergeRequest struct { - ID int `json:"id"` - IID int `json:"iid"` - ProjectID int `json:"project_id"` - Title string `json:"title"` - State string `json:"state"` - CreatedAt *time.Time `json:"created_at"` - UpdatedAt *time.Time `json:"updated_at"` - SourceBranch string `json:"source_branch"` - TargetBranch string `json:"target_branch"` - SourceProjectID int `json:"source_project_id"` - TargetProjectID int `json:"target_project_id"` - Description string `json:"description"` - Author *BasicUser `json:"author"` - Changes []Changes `json:"changes"` -} - -type Changes struct { - OldPath string `json:"old_path"` - NewPath string `json:"new_path"` - AMode string `json:"a_mode"` - BMode string `json:"b_mode"` - Diff string `json:"diff"` - NewFile bool `json:"new_file"` - RenamedFile bool `json:"renamed_file"` - DeletedFile bool `json:"deleted_file"` -} - -type BasicUser struct { - ID int `json:"id"` - Username string `json:"username"` - Name string `json:"name"` - State string `json:"state"` - CreatedAt *time.Time `json:"created_at"` - AvatarURL string `json:"avatar_url"` - WebURL string `json:"web_url"` -} - -func (c *Client) ListOpenedProjectMergeRequests(owner, repo, targetBranch string, log *zap.SugaredLogger) ([]*MergeRequest, error) { - url := fmt.Sprintf("/api/v4/projects/%s/isource/merge_requests", generateProjectName(owner, repo)) - qs := map[string]string{ - "state": "opened", - "per_page": "100", - } - if targetBranch != "" { - qs["target_branch"] = targetBranch - } - - var err error - var mergeRequests []*MergeRequest - if _, err = c.Get(url, httpclient.SetQueryParams(qs), httpclient.SetResult(&mergeRequests)); err != nil { - log.Errorf("Failed to list project mergeRequests, error: %s", err) - return mergeRequests, err - } - - return mergeRequests, nil -} - -func (c *Client) ListChangedFiles(event *MergeEvent) ([]string, error) { - files := make([]string, 0) - - url := fmt.Sprintf("/api/v4/projects/%d/merge_requests/%d/changes", event.ObjectAttributes.TargetProjectID, event.ObjectAttributes.IID) - var err error - var mergeRequest *MergeRequest - if _, err = c.Get(url, httpclient.SetResult(&mergeRequest)); err != nil { - log.Errorf("Failed to get project mergeRequest, error: %s", err) - return files, err - } - - for _, change := range mergeRequest.Changes { - files = append(files, change.NewPath) - files = append(files, change.OldPath) - } - - return files, nil -} - -func (c *Client) GetLatestPRCommitList(projectID string, pr int, log *zap.SugaredLogger) (*Commit, error) { - url := fmt.Sprintf("/api/v4/projects/%s/merge_requests/%d/commits", projectID, pr) - - var commit *Commit - if _, err := c.Get(url, httpclient.SetResult(&commit)); err != nil { - log.Errorf("Failed to get project mergeRequest commits, error: %s", err) - return commit, err - } - return commit, nil -} diff --git a/pkg/tool/ilyshin/project.go b/pkg/tool/ilyshin/project.go deleted file mode 100644 index d348a2da49c9e3ecf478c619810f7f936930b7f5..0000000000000000000000000000000000000000 --- a/pkg/tool/ilyshin/project.go +++ /dev/null @@ -1,80 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ilyshin - -import ( - "fmt" - "time" - - "go.uber.org/zap" - "k8s.io/apimachinery/pkg/util/sets" - - "github.com/koderover/zadig/pkg/tool/httpclient" -) - -type Project struct { - ID int `json:"id"` - Description string `json:"description"` - Name string `json:"name"` - NameWithNamespace string `json:"name_with_namespace"` - Path string `json:"path"` - PathWithNamespace string `json:"path_with_namespace"` - CreatedAt *time.Time `json:"created_at,omitempty"` - Archived bool `json:"archived"` - DefaultBranch string `json:"default_branch"` - Namespace *ProjectNamespace `json:"namespace"` -} - -type ProjectNamespace struct { - ID int `json:"id"` - Name string `json:"name"` - Path string `json:"path"` - Kind string `json:"kind"` - FullPath string `json:"full_path"` -} - -func (c *Client) ListNamespaces(keyword string, log *zap.SugaredLogger) ([]*Project, error) { - url := fmt.Sprintf("/api/v4/projects") - qs := map[string]string{ - "order_by": "name", - "sort": "asc", - "project_type": "group", // group or project - "per_page": "100", - "simple": "true", - } - if keyword != "" && len(keyword) > 2 { - qs["search"] = keyword - } - var err error - var gps []*Project - if _, err = c.Get(url, httpclient.SetQueryParams(qs), httpclient.SetResult(&gps)); err != nil { - log.Errorf("Failed to list projects, error: %s", err) - return gps, err - } - - var resp []*Project - projectNames := sets.NewString() - for _, gp := range gps { - if projectNames.Has(gp.Namespace.Name) || gp.Namespace.Kind == "user" { - continue - } - - projectNames.Insert(gp.Namespace.Name) - resp = append(resp, gp) - } - return resp, nil -} diff --git a/pkg/tool/ilyshin/repo.go b/pkg/tool/ilyshin/repo.go deleted file mode 100644 index 1bd95db412c267c5129bf1ec6c8f8bf7e697e642..0000000000000000000000000000000000000000 --- a/pkg/tool/ilyshin/repo.go +++ /dev/null @@ -1,122 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ilyshin - -import ( - "fmt" - - "github.com/koderover/zadig/pkg/tool/httpclient" - "github.com/koderover/zadig/pkg/tool/log" -) - -type TreeNode struct { - ID string `json:"id"` - Name string `json:"name"` - Type string `json:"type"` - Path string `json:"path"` - Mode string `json:"mode"` -} - -type File struct { - FileName string `json:"file_name"` - FilePath string `json:"file_path"` - Size int `json:"size"` - Encoding string `json:"encoding"` - Content string `json:"content"` - Ref string `json:"ref"` - BlobID string `json:"blob_id"` - CommitID string `json:"commit_id"` - SHA256 string `json:"content_sha256"` - LastCommitID string `json:"last_commit_id"` -} - -type Diff struct { - Diff string `json:"diff"` - NewPath string `json:"new_path"` - OldPath string `json:"old_path"` - AMode string `json:"a_mode"` - BMode string `json:"b_mode"` - NewFile bool `json:"new_file"` - RenamedFile bool `json:"renamed_file"` - DeletedFile bool `json:"deleted_file"` -} - -func (c *Client) ListTree(owner, repo string, ref string, path string) ([]*TreeNode, error) { - url := fmt.Sprintf("/api/v4/projects/%s/repository/tree", generateProjectName(owner, repo)) - qs := map[string]string{ - "ref": ref, - "path": path, - "per_page": "100", - } - - var err error - var treeNodes []*TreeNode - if _, err = c.Get(url, httpclient.SetQueryParams(qs), httpclient.SetResult(&treeNodes)); err != nil { - log.Errorf("Failed to list project tree nodes, error: %s", err) - return treeNodes, err - } - - return treeNodes, nil -} - -func (c *Client) GetRawFile(owner, repo string, sha string, fileName string) ([]byte, error) { - File, err := c.GetFile(owner, repo, sha, fileName) - if err != nil { - return nil, err - } - var resp []byte - url := fmt.Sprintf("/api/v4/projects/%s/repository/blobs/%s/raw", generateProjectName(owner, repo), File.BlobID) - if _, err = c.Get(url, httpclient.SetResult(&resp)); err != nil { - log.Errorf("Failed to get project blob raw, error: %s", err) - return resp, err - } - - return resp, nil -} - -func (c *Client) GetFile(owner, repo string, ref, path string) (*File, error) { - url := fmt.Sprintf("/api/v4/projects/%s/repository/files/%s", generateProjectName(owner, repo), path) - qs := map[string]string{ - "ref": ref, - } - - var err error - var file *File - if _, err = c.Get(url, httpclient.SetQueryParams(qs), httpclient.SetResult(&file)); err != nil { - log.Errorf("Failed to get project file content, error: %s", err) - return file, err - } - - return file, nil -} - -func (c *Client) Compare(projectID int, from, to string) ([]*Diff, error) { - url := fmt.Sprintf("/api/v4/projects/%d/repository/compare", projectID) - qs := map[string]string{ - "from": from, - "to": to, - } - - var err error - var diffs []*Diff - if _, err = c.Get(url, httpclient.SetQueryParams(qs), httpclient.SetResult(&diffs)); err != nil { - log.Errorf("Failed to compare content, error: %s", err) - return diffs, err - } - - return diffs, nil -} diff --git a/pkg/tool/ilyshin/webhook.go b/pkg/tool/ilyshin/webhook.go deleted file mode 100644 index 2d38c4d25a834fe4c1ec3e4bb1ffcead8be2d260..0000000000000000000000000000000000000000 --- a/pkg/tool/ilyshin/webhook.go +++ /dev/null @@ -1,100 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ilyshin - -import ( - "fmt" - "time" - - "go.uber.org/zap" - - "github.com/koderover/zadig/pkg/tool/httpclient" - "github.com/koderover/zadig/pkg/tool/log" -) - -type ProjectHook struct { - ID int `json:"id"` - URL string `json:"url"` - ConfidentialNoteEvents bool `json:"confidential_note_events"` - ProjectID int `json:"project_id"` - PushEvents bool `json:"push_events"` - PushEventsBranchFilter string `json:"push_events_branch_filter"` - IssuesEvents bool `json:"issues_events"` - ConfidentialIssuesEvents bool `json:"confidential_issues_events"` - MergeRequestsEvents bool `json:"merge_requests_events"` - TagPushEvents bool `json:"tag_push_events"` - NoteEvents bool `json:"note_events"` - JobEvents bool `json:"job_events"` - PipelineEvents bool `json:"pipeline_events"` - WikiPageEvents bool `json:"wiki_page_events"` - DeploymentEvents bool `json:"deployment_events"` - ReleasesEvents bool `json:"releases_events"` - EnableSSLVerification bool `json:"enable_ssl_verification"` - CreatedAt *time.Time `json:"created_at"` -} - -type Hook struct { - URL string `url:"url,omitempty" json:"url,omitempty"` - Name string `url:"name,omitempty" json:"name,omitempty"` - Token string `url:"token,omitempty" json:"token,omitempty"` - PushEvents bool `url:"push_events,omitempty" json:"push_events,omitempty"` - MergeRequestsEvents bool `url:"merge_requests_events,omitempty" json:"merge_requests_events,omitempty"` - TagPushEvents bool `url:"tag_push_events,omitempty" json:"tag_push_events,omitempty"` - EnableSSLVerification bool `url:"enable_ssl_verification,omitempty" json:"enable_ssl_verification,omitempty"` -} - -func (c *Client) ListProjectHooks(owner, repo string, log *zap.SugaredLogger) ([]*ProjectHook, error) { - url := fmt.Sprintf("/api/v4/projects/%s/hooks", generateProjectName(owner, repo)) - qs := map[string]string{ - "per_page": "100", - } - - var err error - var projectHooks []*ProjectHook - if _, err = c.Get(url, httpclient.SetQueryParams(qs), httpclient.SetResult(&projectHooks)); err != nil { - log.Errorf("Failed to list project hooks, error: %s", err) - return projectHooks, err - } - - return projectHooks, nil -} - -func (c *Client) AddProjectHook(owner, repo, hookURL, token string) (*ProjectHook, error) { - url := fmt.Sprintf("/api/v4/projects/%s/hooks", generateProjectName(owner, repo)) - hook := &Hook{ - URL: hookURL, - Token: token, - MergeRequestsEvents: true, - PushEvents: true, - TagPushEvents: true, - } - var projectHook *ProjectHook - if _, err := c.Post(url, httpclient.SetBody(hook), httpclient.SetResult(&projectHook)); err != nil { - log.Errorf("Failed to create project hook, error: %s", err) - return projectHook, err - } - return projectHook, nil -} - -func (c *Client) DeleteProjectHook(owner, repo string, id int) error { - url := fmt.Sprintf("/api/v4/projects/%s/hooks/%d", generateProjectName(owner, repo), id) - if _, err := c.Delete(url); err != nil { - log.Errorf("Failed to delete project webhook, error: %s", err) - return err - } - return nil -} diff --git a/pkg/tool/kube/client/cluster.go b/pkg/tool/kube/client/cluster.go index 927cd411525ecff7c5352bbc133ab7f88181a7b8..bba4563529f13d747840f9fefd5ee5dd92e53c41 100644 --- a/pkg/tool/kube/client/cluster.go +++ b/pkg/tool/kube/client/cluster.go @@ -22,6 +22,8 @@ import ( "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -38,12 +40,32 @@ var c cluster.Cluster // Cluster is a singleton, it will be initialized only once. func Cluster() cluster.Cluster { once.Do(func() { - c = initCluster(ctrl.GetConfigOrDie()) + var err error + c, err = initCluster(ctrl.GetConfigOrDie()) + if err != nil { + panic(err) + } }) return c } +func NewClientSet() (*kubernetes.Clientset, error) { + config, err := rest.InClusterConfig() + if err != nil { + return nil, err + } + return kubernetes.NewForConfig(config) +} + +func NewDynamicClient() (dynamic.Interface, error) { + config, err := rest.InClusterConfig() + if err != nil { + return nil, err + } + return dynamic.NewForConfig(config) +} + func Client() client.Client { return Cluster().GetClient() } @@ -77,7 +99,10 @@ func NewClientFromAPIConfig(cfg *api.Config) (client.Client, error) { return nil, err } - cls := initCluster(restConfig) + cls, err := initCluster(restConfig) + if err != nil { + return nil, err + } return newAPIClient(cls.GetClient(), cls.GetAPIReader()), nil } @@ -105,7 +130,7 @@ func (c *apiClient) List(ctx context.Context, list client.ObjectList, opts ...cl return c.apiReader.List(ctx, list, opts...) } -func initCluster(restConfig *rest.Config) cluster.Cluster { +func initCluster(restConfig *rest.Config) (cluster.Cluster, error) { scheme := runtime.NewScheme() // add all known types @@ -116,8 +141,8 @@ func initCluster(restConfig *rest.Config) cluster.Cluster { clusterOptions.Scheme = scheme }) if err != nil { - panic(errors.Wrap(err, "unable to init client")) + return nil, errors.Wrap(err, "unable to init client") } - return c + return c, nil } diff --git a/pkg/tool/kube/containerlog/log.go b/pkg/tool/kube/containerlog/log.go index 305c97c33e4489b3a28bf518a518ade45690930f..36361070c403db1ca7ca3c93b621860d3109e9ce 100644 --- a/pkg/tool/kube/containerlog/log.go +++ b/pkg/tool/kube/containerlog/log.go @@ -18,15 +18,37 @@ package containerlog import ( "context" + "fmt" "io" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + + "github.com/koderover/zadig/pkg/tool/log" ) func GetContainerLogs(namespace, podName, containerName string, follow bool, tailLines int64, out io.Writer, clientset kubernetes.Interface) error { readCloser, err := GetContainerLogStream(context.TODO(), namespace, podName, containerName, follow, tailLines, clientset) if err != nil { + log.Warnf("Failed to get pod log from stream: %s. Try to get logs from pod object.", err) + + // For Serverless K8s, we may not be able to get logs from Pod in the Failed state, but we can configure + // `container.terminationMessagePolicy=FallbackToLogsOnError` to get the latest exception information from the Pod Object. + pod, err := clientset.CoreV1().Pods(namespace).Get(context.TODO(), podName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get pod %s in %s: %s", podName, namespace, err) + } + + // Note: Do two protections to avoid panic. + if pod.Status.Phase == corev1.PodPending || pod.Status.Phase == corev1.PodUnknown { + return fmt.Errorf("phase of Pod %s in ns %s is %s", pod.Name, pod.Namespace, pod.Status.Phase) + } + if len(pod.Status.ContainerStatuses) == 0 { + return fmt.Errorf("length of container statuses is 0 for pod %s in ns %s", podName, namespace) + } + + _, err = out.Write([]byte(pod.Status.ContainerStatuses[0].State.Terminated.Message)) return err } diff --git a/pkg/tool/kube/getter/base.go b/pkg/tool/kube/getter/base.go index d40e5a1a89b9b2912dda3b003ee9644e313f2770..eee6f183ab4f55a8625e08fbf058cd5e46c72606 100644 --- a/pkg/tool/kube/getter/base.go +++ b/pkg/tool/kube/getter/base.go @@ -20,6 +20,7 @@ import ( "context" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" @@ -111,6 +112,45 @@ func GetResourceYamlInCache(ns, name string, gvk schema.GroupVersionKind, cl cli return data, true, nil } +func GetResourceJSONInCacheFormat(ns, name string, gvk schema.GroupVersionKind, cl client.Reader) ([]byte, bool, error) { + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(gvk) + + found, err := GetResourceInCache(ns, name, u, cl) + if err != nil || !found { + return nil, false, err + } + u.SetManagedFields(nil) + u.SetUID("") + u.SetSelfLink("") + u.SetResourceVersion("") + u.SetCreationTimestamp(metav1.Time{}) + u.SetNamespace("") + u.SetGeneration(0) + content := u.UnstructuredContent() + delete(content, "status") + u.SetUnstructuredContent(content) + data, err := u.MarshalJSON() + if err != nil { + return nil, false, err + } + + return data, true, nil +} + +func GetResourceYamlInCacheFormat(ns, name string, gvk schema.GroupVersionKind, cl client.Reader) ([]byte, bool, error) { + d, found, err := GetResourceJSONInCacheFormat(ns, name, gvk, cl) + if err != nil || !found || len(d) == 0 { + return nil, false, err + } + + data, err := yaml.JSONToYAML(d) + if err != nil { + return nil, false, err + } + return data, true, nil +} + // ListResourceJSONInCache gets a set of specific Kubernetes object in local cache, and return a representation in json format. func ListResourceJSONInCache(ns string, selector labels.Selector, fieldSelector fields.Selector, gvk schema.GroupVersionKind, cl client.Reader) ([][]byte, error) { u := &unstructured.UnstructuredList{} @@ -145,6 +185,7 @@ func ListResourceYamlInCache(ns string, selector labels.Selector, fieldSelector var res [][]byte for _, item := range u.Items { + item.SetManagedFields(nil) data, err := item.MarshalJSON() if err != nil { return nil, err diff --git a/pkg/tool/kube/getter/configmap.go b/pkg/tool/kube/getter/configmap.go index 61b846b9466ede39e57be9e95e41387e8850f64b..dca102c125fd83bf8eeea43d9011c58b40688160 100644 --- a/pkg/tool/kube/getter/configmap.go +++ b/pkg/tool/kube/getter/configmap.go @@ -25,6 +25,12 @@ import ( func ListConfigMaps(ns string, selector labels.Selector, cl client.Client) ([]*corev1.ConfigMap, error) { l := &corev1.ConfigMapList{} + gvk := schema.GroupVersionKind{ + Group: "core", + Kind: "ConfigMap", + Version: "v1", + } + l.SetGroupVersionKind(gvk) err := ListResourceInCache(ns, selector, nil, l, cl) if err != nil { return nil, err @@ -55,3 +61,21 @@ func ListConfigMapsYaml(ns string, selector labels.Selector, cl client.Client) ( } return ListResourceYamlInCache(ns, selector, nil, gvk, cl) } + +func GetConfigMapYaml(ns string, name string, cl client.Client) ([]byte, bool, error) { + gvk := schema.GroupVersionKind{ + Group: "", + Kind: "ConfigMap", + Version: "v1", + } + return GetResourceYamlInCache(ns, name, gvk, cl) +} + +func GetConfigMapYamlFormat(ns string, name string, cl client.Client) ([]byte, bool, error) { + gvk := schema.GroupVersionKind{ + Group: "", + Kind: "ConfigMap", + Version: "v1", + } + return GetResourceYamlInCacheFormat(ns, name, gvk, cl) +} diff --git a/pkg/tool/kube/getter/deployment.go b/pkg/tool/kube/getter/deployment.go index 3a343d2d532098819233b5c6b330c0a55c2647ad..d2eead6d3641aa1d7d12e918377738b1178a9427 100644 --- a/pkg/tool/kube/getter/deployment.go +++ b/pkg/tool/kube/getter/deployment.go @@ -20,6 +20,7 @@ import ( appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/informers" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -47,6 +48,13 @@ func ListDeployments(ns string, selector labels.Selector, cl client.Client) ([]* return res, err } +func ListDeploymentsWithCache(selector labels.Selector, lister informers.SharedInformerFactory) ([]*appsv1.Deployment, error) { + if selector == nil { + selector = labels.NewSelector() + } + return lister.Apps().V1().Deployments().Lister().List(selector) +} + func ListDeploymentsYaml(ns string, selector labels.Selector, cl client.Client) ([][]byte, error) { gvk := schema.GroupVersionKind{ Group: "apps", @@ -64,3 +72,12 @@ func GetDeploymentYaml(ns string, name string, cl client.Client) ([]byte, bool, } return GetResourceYamlInCache(ns, name, gvk, cl) } + +func GetDeploymentYamlFormat(ns string, name string, cl client.Client) ([]byte, bool, error) { + gvk := schema.GroupVersionKind{ + Group: "apps", + Kind: "Deployment", + Version: "v1", + } + return GetResourceYamlInCacheFormat(ns, name, gvk, cl) +} diff --git a/pkg/tool/kube/getter/ingress.go b/pkg/tool/kube/getter/ingress.go index 04ef661fe175650482b54c5192381172650d4158..2813a489acd165e5d3e882f1219db06b5d65a157 100644 --- a/pkg/tool/kube/getter/ingress.go +++ b/pkg/tool/kube/getter/ingress.go @@ -18,34 +18,69 @@ package getter import ( extensionsv1beta1 "k8s.io/api/extensions/v1beta1" + v1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/pkg/apis/extensions" "sigs.k8s.io/controller-runtime/pkg/client" + + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" ) -func GetIngress(ns, name string, cl client.Client) (*extensionsv1beta1.Ingress, bool, error) { - g := &extensionsv1beta1.Ingress{} - found, err := GetResourceInCache(ns, name, g, cl) - if err != nil || !found { - g = nil +func GetExtensionsV1Beta1Ingress(namespace, name string, lister informers.SharedInformerFactory) (*extensionsv1beta1.Ingress, bool, error) { + ret, err := lister.Extensions().V1beta1().Ingresses().Lister().Ingresses(namespace).Get(name) + if err == nil { + return ret, true, nil } + return nil, false, err +} - return g, found, err +func GetNetworkingV1Ingress(namespace, name string, lister informers.SharedInformerFactory) (*v1.Ingress, error) { + return lister.Networking().V1().Ingresses().Lister().Ingresses(namespace).Get(name) } -func ListIngresses(ns string, selector labels.Selector, cl client.Client) ([]*extensionsv1beta1.Ingress, error) { - // ingress is moved to networking.k8s.io/v1beta1 from extensions/v1beta1. - is := &extensionsv1beta1.IngressList{} - err := ListResourceInCache(ns, selector, nil, is, cl) +func GetUnstructuredIngress(namespace, name string, cl client.Client, clientset *kubernetes.Clientset) (*unstructured.Unstructured, bool, error) { + version, err := clientset.Discovery().ServerVersion() if err != nil { - return nil, err + return nil, false, err + } + gvk := schema.GroupVersionKind{ + Group: "extensions", + Kind: "Ingress", + Version: "v1beta1", } + if !kubeclient.VersionLessThan122(version) { + gvk = schema.GroupVersionKind{ + Group: "networking.k8s.io", + Kind: "Ingress", + Version: "v1", + } + } + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(gvk) + found, err := GetResourceInCache(namespace, name, u, cl) + if err != nil || !found { + u = nil + } + return u, found, err +} - var res []*extensionsv1beta1.Ingress - for i := range is.Items { - res = append(res, &is.Items[i]) +// ListExtensionsV1Beta1Ingresses gets the ingress (extensions/v1beta1) from the informer +func ListExtensionsV1Beta1Ingresses(selector labels.Selector, lister informers.SharedInformerFactory) ([]*extensionsv1beta1.Ingress, error) { + if selector == nil { + selector = labels.NewSelector() } - return res, err + return lister.Extensions().V1beta1().Ingresses().Lister().List(selector) +} + +func ListNetworkingV1Ingress(selector labels.Selector, lister informers.SharedInformerFactory) ([]*v1.Ingress, error) { + if selector == nil { + selector = labels.NewSelector() + } + return lister.Networking().V1().Ingresses().Lister().List(selector) } func ListIngressesYaml(ns string, selector labels.Selector, cl client.Client) ([][]byte, error) { @@ -56,3 +91,72 @@ func ListIngressesYaml(ns string, selector labels.Selector, cl client.Client) ([ } return ListResourceYamlInCache(ns, selector, nil, gvk, cl) } + +func ListIngresses(namespace string, cl client.Client, lessThan122 bool) (*unstructured.UnstructuredList, error) { + gvk := schema.GroupVersionKind{ + Group: "extensions", + Kind: "Ingress", + Version: "v1beta1", + } + if !lessThan122 { + gvk = schema.GroupVersionKind{ + Group: "networking.k8s.io", + Kind: "Ingress", + Version: "v1", + } + } + u := &unstructured.UnstructuredList{} + u.SetGroupVersionKind(gvk) + + err := ListResourceInCache(namespace, labels.Everything(), nil, u, cl) + if err != nil { + return u, err + } + return u, err +} + +func ListIngressesFormat(namespace string, cl client.Client, lessThan122 bool) ([]*extensions.Ingress, error) { + l := &extensions.IngressList{} + gvk := schema.GroupVersionKind{ + Group: "extensions", + Kind: "Ingress", + Version: "v1beta1", + } + if !lessThan122 { + gvk = schema.GroupVersionKind{ + Group: "networking.k8s.io", + Kind: "Ingress", + Version: "v1", + } + } + u := &unstructured.UnstructuredList{} + u.SetGroupVersionKind(gvk) + + err := ListResourceInCache(namespace, labels.Everything(), nil, u, cl) + if err != nil { + return nil, err + } + var res []*extensions.Ingress + for i := range l.Items { + res = append(res, &l.Items[i]) + } + return res, err +} + +func GetIngressYaml(ns string, name string, cl client.Client) ([]byte, bool, error) { + gvk := schema.GroupVersionKind{ + Group: "extensions", + Kind: "Ingress", + Version: "v1beta1", + } + return GetResourceYamlInCache(ns, name, gvk, cl) +} + +func GetIngressYamlFormat(ns string, name string, cl client.Client) ([]byte, bool, error) { + gvk := schema.GroupVersionKind{ + Group: "extensions", + Kind: "Ingress", + Version: "v1beta1", + } + return GetResourceYamlInCacheFormat(ns, name, gvk, cl) +} diff --git a/pkg/tool/kube/getter/pod.go b/pkg/tool/kube/getter/pod.go index 8927b4fdbfe3544fecbe753dd27a70fd7215053e..0153f459f4a45696f31aa1edb64dc1674914ffef 100644 --- a/pkg/tool/kube/getter/pod.go +++ b/pkg/tool/kube/getter/pod.go @@ -19,6 +19,7 @@ package getter import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/informers" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -45,3 +46,11 @@ func ListPods(ns string, selector labels.Selector, cl client.Client) ([]*corev1. } return res, err } + +func ListPodsWithCache(selector labels.Selector, informer informers.SharedInformerFactory) ([]*corev1.Pod, error) { + if selector == nil { + selector = labels.NewSelector() + } + + return informer.Core().V1().Pods().Lister().List(selector) +} diff --git a/pkg/tool/kube/getter/rolebinding.go b/pkg/tool/kube/getter/pvc.go similarity index 37% rename from pkg/tool/kube/getter/rolebinding.go rename to pkg/tool/kube/getter/pvc.go index 1314a663c7bb6284413ab1ddcd48395491a83f08..0d1a3fb034fd97c556a8b29378accfc4809c62e4 100644 --- a/pkg/tool/kube/getter/rolebinding.go +++ b/pkg/tool/kube/getter/pvc.go @@ -17,16 +17,55 @@ limitations under the License. package getter import ( - rbacv1beta1 "k8s.io/api/rbac/v1beta1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/client" ) -func GetRoleBinding(ns, name string, cl client.Client) (*rbacv1beta1.RoleBinding, bool, error) { - g := &rbacv1beta1.RoleBinding{} +func ListPvcs(ns string, selector fields.Selector, cl client.Reader) ([]*corev1.PersistentVolumeClaim, error) { + pvcList := &corev1.PersistentVolumeClaimList{} + gvk := schema.GroupVersionKind{ + Group: "core", + Kind: "PersistentVolumeClaim", + Version: "v1", + } + pvcList.SetGroupVersionKind(gvk) + err := ListResourceInCache(ns, nil, selector, pvcList, cl) + if err != nil { + return nil, err + } + + var res []*corev1.PersistentVolumeClaim + for i := range pvcList.Items { + res = append(res, &pvcList.Items[i]) + } + return res, err +} + +func GetPVCYaml(ns string, name string, cl client.Client) ([]byte, bool, error) { + gvk := schema.GroupVersionKind{ + Group: "", + Kind: "PersistentVolumeClaim", + Version: "v1", + } + return GetResourceYamlInCache(ns, name, gvk, cl) +} + +func GetPvc(ns, name string, cl client.Client) (*corev1.PersistentVolumeClaim, bool, error) { + g := &corev1.PersistentVolumeClaim{} found, err := GetResourceInCache(ns, name, g, cl) if err != nil || !found { g = nil } - return g, found, err } + +func GetPVCYamlFormat(ns string, name string, cl client.Client) ([]byte, bool, error) { + gvk := schema.GroupVersionKind{ + Group: "", + Kind: "PersistentVolumeClaim", + Version: "v1", + } + return GetResourceYamlInCacheFormat(ns, name, gvk, cl) +} diff --git a/pkg/tool/kube/getter/secrets.go b/pkg/tool/kube/getter/secrets.go index 1eaee5ad166ee41c4f0dc17ac122770c2623a25d..14d060b3f2c89af4062741b131011b38dcec1040 100644 --- a/pkg/tool/kube/getter/secrets.go +++ b/pkg/tool/kube/getter/secrets.go @@ -18,6 +18,7 @@ package getter import ( corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -30,3 +31,41 @@ func GetSecret(ns, name string, cl client.Client) (*corev1.Secret, bool, error) return svc, found, err } + +func ListSecrets(ns string, cl client.Client) ([]*corev1.Secret, error) { + l := &corev1.SecretList{} + gvk := schema.GroupVersionKind{ + Group: "core", + Kind: "Secret", + Version: "v1", + } + l.SetGroupVersionKind(gvk) + err := ListResourceInCache(ns, nil, nil, l, cl) + if err != nil { + return nil, err + } + + var res []*corev1.Secret + for i := range l.Items { + res = append(res, &l.Items[i]) + } + return res, err +} + +func GetSecretYaml(ns string, name string, cl client.Client) ([]byte, bool, error) { + gvk := schema.GroupVersionKind{ + Group: "", + Kind: "Secret", + Version: "v1", + } + return GetResourceYamlInCache(ns, name, gvk, cl) +} + +func GetSecretYamlFormat(ns string, name string, cl client.Client) ([]byte, bool, error) { + gvk := schema.GroupVersionKind{ + Group: "", + Kind: "Secret", + Version: "v1", + } + return GetResourceYamlInCacheFormat(ns, name, gvk, cl) +} diff --git a/pkg/tool/kube/getter/service.go b/pkg/tool/kube/getter/service.go index 1cfb554969419128f9a73a03b2b9562f746f4f6c..593dd2ff41e8b82619fb54f06bdafe914919da77 100644 --- a/pkg/tool/kube/getter/service.go +++ b/pkg/tool/kube/getter/service.go @@ -20,6 +20,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/informers" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -47,6 +48,13 @@ func ListServices(ns string, selector labels.Selector, cl client.Client) ([]*cor return res, err } +func ListServicesWithCache(selector labels.Selector, lister informers.SharedInformerFactory) ([]*corev1.Service, error) { + if selector == nil { + selector = labels.NewSelector() + } + return lister.Core().V1().Services().Lister().List(selector) +} + func ListServicesYaml(ns string, selector labels.Selector, cl client.Client) ([][]byte, error) { gvk := schema.GroupVersionKind{ Group: "", @@ -55,3 +63,21 @@ func ListServicesYaml(ns string, selector labels.Selector, cl client.Client) ([] } return ListResourceYamlInCache(ns, selector, nil, gvk, cl) } + +func GetServiceYaml(ns string, name string, cl client.Client) ([]byte, bool, error) { + gvk := schema.GroupVersionKind{ + Group: "", + Kind: "Service", + Version: "v1", + } + return GetResourceYamlInCache(ns, name, gvk, cl) +} + +func GetServiceYamlFormat(ns string, name string, cl client.Client) ([]byte, bool, error) { + gvk := schema.GroupVersionKind{ + Group: "", + Kind: "Service", + Version: "v1", + } + return GetResourceYamlInCacheFormat(ns, name, gvk, cl) +} diff --git a/pkg/tool/kube/getter/statefulset.go b/pkg/tool/kube/getter/statefulset.go index 31e3a84f75375aaaaa15da76709ae0e7a9c4f871..8b96976d73f7070397c3b4f3b7763e0877dc8a43 100644 --- a/pkg/tool/kube/getter/statefulset.go +++ b/pkg/tool/kube/getter/statefulset.go @@ -20,6 +20,7 @@ import ( appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/informers" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -47,6 +48,13 @@ func ListStatefulSets(ns string, selector labels.Selector, cl client.Client) ([] return res, err } +func ListStatefulSetsWithCache(selector labels.Selector, lister informers.SharedInformerFactory) ([]*appsv1.StatefulSet, error) { + if selector == nil { + selector = labels.NewSelector() + } + return lister.Apps().V1().StatefulSets().Lister().List(selector) +} + func ListStatefulSetsYaml(ns string, selector labels.Selector, cl client.Client) ([][]byte, error) { gvk := schema.GroupVersionKind{ Group: "apps", @@ -64,3 +72,12 @@ func GetStatefulSetYaml(ns string, name string, cl client.Client) ([]byte, bool, } return GetResourceYamlInCache(ns, name, gvk, cl) } + +func GetStatefulSetYamlFormat(ns string, name string, cl client.Client) ([]byte, bool, error) { + gvk := schema.GroupVersionKind{ + Group: "apps", + Kind: "StatefulSet", + Version: "v1", + } + return GetResourceYamlInCacheFormat(ns, name, gvk, cl) +} diff --git a/pkg/tool/kube/informer/informer.go b/pkg/tool/kube/informer/informer.go new file mode 100644 index 0000000000000000000000000000000000000000..58c9169d03451cba0cd9680ea94b4470e8ca1cf4 --- /dev/null +++ b/pkg/tool/kube/informer/informer.go @@ -0,0 +1,104 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package informer + +import ( + "fmt" + "sync" + "time" + + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + + "github.com/koderover/zadig/pkg/setting" +) + +var InformersMap sync.Map +var StopChanMap sync.Map + +// NewInformer initialize and start an informer for specific namespace in given cluster. +// Currently the informer will NOT stop unless the service is down +// If you want to watch a new resource, remember to register here +// Current list: +// - Deployment +// - StatefulSet +// - Service +// - Pod +// - Ingress (extentions/v1beta1) <- as of version 1.9.0, this is the resource we watch +func NewInformer(clusterID, namespace string, cls *kubernetes.Clientset) (informers.SharedInformerFactory, error) { + // this is a stupid compatibility code + if clusterID == "" { + clusterID = setting.LocalClusterID + } + key := generateInformerKey(clusterID, namespace) + if informer, ok := InformersMap.Load(key); ok { + return informer.(informers.SharedInformerFactory), nil + } + opts := informers.WithNamespace(namespace) + informerFactory := informers.NewSharedInformerFactoryWithOptions(cls, time.Minute, opts) + // register the resources to be watched + informerFactory.Apps().V1().Deployments().Lister() + informerFactory.Apps().V1().StatefulSets().Lister() + informerFactory.Core().V1().Services().Lister() + informerFactory.Core().V1().Pods().Lister() + versionInfo, err := cls.Discovery().ServerVersion() + if err != nil { + return nil, err + } + // if less than v1.22.0, then we look for the extensions/v1beta1 ingress + if kubeclient.VersionLessThan122(versionInfo) { + informerFactory.Extensions().V1beta1().Ingresses().Lister() + } else { + // otherwise above resource is deprecated, we watch for the k8s.networking.io/v1 ingress + informerFactory.Networking().V1().Ingresses().Lister() + } + + // stop channel will be stored for future stop + stopchan := make(chan struct{}) + informerFactory.Start(stopchan) + // wait for the cache to be synced for the first time + informerFactory.WaitForCacheSync(make(chan struct{})) + // in case there is a concurrent situation, we find if there is one informer in the map + if _, ok := InformersMap.Load(key); ok { + // if we found that stop channel + if stopchan, ok := StopChanMap.Load(key); ok { + close(stopchan.(chan struct{})) + StopChanMap.Delete(key) + } + } + InformersMap.Store(key, informerFactory) + StopChanMap.Store(key, stopchan) + return informerFactory, nil +} + +func DeleteInformer(clusterID, namespace string) { + key := generateInformerKey(clusterID, namespace) + // if informer exists + if _, ok := InformersMap.Load(key); ok { + // if we found that stop channel + if stopchan, ok := StopChanMap.Load(key); ok { + close(stopchan.(chan struct{})) + StopChanMap.Delete(key) + } + } + InformersMap.Delete(key) +} + +func generateInformerKey(clusterID, namespace string) string { + return fmt.Sprintf(setting.InformerNamingConvention, clusterID, namespace) +} diff --git a/pkg/tool/kube/multicluster/service.go b/pkg/tool/kube/multicluster/service.go index 067c8740fbb3bd6c5356cc3812228dac297a6ba1..ff0028b41fae5f742655e38b3f1e761d9677d7e1 100644 --- a/pkg/tool/kube/multicluster/service.go +++ b/pkg/tool/kube/multicluster/service.go @@ -23,6 +23,7 @@ import ( "github.com/gin-gonic/gin" "github.com/pkg/errors" "go.uber.org/zap" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd/api" @@ -44,6 +45,32 @@ func GetKubeClient(hubServerAddr, clusterID string) (client.Client, error) { return clusterService.GetKubeClient(clusterID) } +func GetKubeClientSet(hubServerAddr, clusterID string) (*kubernetes.Clientset, error) { + if clusterID == "" { + return krkubeclient.NewClientSet() + } + + clusterService, err := NewAgent(hubServerAddr) + if err != nil { + return nil, fmt.Errorf("failed to get clusterService: %v", err) + } + + return clusterService.GetClientGoKubeClient(clusterID) +} + +func GetDynamicKubeclient(hubServerAddr, clusterID string) (dynamic.Interface, error) { + if clusterID == "" { + return krkubeclient.NewDynamicClient() + } + + clusterService, err := NewAgent(hubServerAddr) + if err != nil { + return nil, fmt.Errorf("failed to get clusterService: %v", err) + } + + return clusterService.GetDynamicKubeClient(clusterID) +} + func GetKubeAPIReader(hubServerAddr, clusterID string) (client.Reader, error) { if clusterID == "" { return krkubeclient.APIReader(), nil @@ -126,6 +153,16 @@ func (s *Agent) GetKubeClient(clusterID string) (client.Client, error) { return krkubeclient.NewClientFromAPIConfig(generateAPIConfig(clusterID, s.hubServerAddr)) } +func (s *Agent) GetClientGoKubeClient(clusterID string) (*kubernetes.Clientset, error) { + config := generateRestConfig(clusterID, s.hubServerAddr) + return kubernetes.NewForConfig(config) +} + +func (s *Agent) GetDynamicKubeClient(clusterID string) (dynamic.Interface, error) { + config := generateRestConfig(clusterID, s.hubServerAddr) + return dynamic.NewForConfig(config) +} + func (s *Agent) ClusterConnected(clusterID string) bool { if err := s.hubClient.HasSession(clusterID); err != nil { return false @@ -164,3 +201,12 @@ func generateAPIConfig(clusterID, hubServerAddr string) *api.Config { CurrentContext: "hubserver", } } + +func generateRestConfig(clusterID, hubServerAddr string) *rest.Config { + return &rest.Config{ + Host: fmt.Sprintf("%s/kube/%s", hubServerAddr, clusterID), + TLSClientConfig: rest.TLSClientConfig{ + Insecure: true, + }, + } +} diff --git a/pkg/tool/kube/podexec/exec.go b/pkg/tool/kube/podexec/exec.go index 5adcb8302510db4e48e843e729b99fcfee6b41a3..d00edc54200c6f2ee7f98fde08657015ce81cf8e 100644 --- a/pkg/tool/kube/podexec/exec.go +++ b/pkg/tool/kube/podexec/exec.go @@ -23,7 +23,9 @@ import ( "strings" corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/remotecommand" "k8s.io/utils/exec" @@ -45,9 +47,13 @@ type ExecOptions struct { // returning stdout, stderr and error. `options` allowed for // additional parameters to be passed. func ExecWithOptions(options ExecOptions) (string, string, bool, error) { + return KubeExec(krkubeclient.Clientset(), krkubeclient.RESTConfig(), options) +} + +func KubeExec(kclient kubernetes.Interface, restConfig *rest.Config, options ExecOptions) (string, string, bool, error) { const tty = false - req := krkubeclient.Clientset().CoreV1().RESTClient().Post(). + req := kclient.CoreV1().RESTClient().Post(). Resource("pods"). Name(options.PodName). Namespace(options.Namespace). @@ -62,7 +68,7 @@ func ExecWithOptions(options ExecOptions) (string, string, bool, error) { TTY: tty, }, scheme.ParameterCodec) - executor, err := remotecommand.NewSPDYExecutor(krkubeclient.RESTConfig(), http.MethodPost, req.URL()) + executor, err := remotecommand.NewSPDYExecutor(restConfig, http.MethodPost, req.URL()) if err != nil { return "", "", false, err } diff --git a/pkg/tool/kube/serializer/decoder_test.go b/pkg/tool/kube/serializer/decoder_test.go index efe0eccc51d02a926be691e1bf66f9e8cfb09de0..0685b3c1ae209caf1f373edf957a050c22900901 100644 --- a/pkg/tool/kube/serializer/decoder_test.go +++ b/pkg/tool/kube/serializer/decoder_test.go @@ -42,7 +42,7 @@ spec: tier: backend spec: containers: - - image: ccr.ccs.tencentyun.com/koderover-public/library-postgres:9.6 + - image: koderover.tencentcloudcr.com/koderover-public/library-postgres:9.6 imagePullPolicy: IfNotPresent name: postgres ports: diff --git a/pkg/tool/kube/updater/base.go b/pkg/tool/kube/updater/base.go index 0c138e62735754d8f352c813ceb3b65de48f43b9..a2ddd97932b457ab2819d19c18f577e91e6f1aa8 100644 --- a/pkg/tool/kube/updater/base.go +++ b/pkg/tool/kube/updater/base.go @@ -79,6 +79,10 @@ func createObject(obj client.Object, cl client.Client) error { return cl.Create(context.TODO(), obj) } +func createObjectNeverAnnotation(obj client.Object, cl client.Client) error { + return cl.Create(context.TODO(), obj) +} + func updateOrCreateObject(obj client.Object, cl client.Client) error { err := updateObject(obj, cl) if err != nil { @@ -122,6 +126,33 @@ func createOrPatchObject(modified client.Object, cl client.Client) error { return patchObjectWithType(current.(client.Object), patchBytes, patchType, cl) } +func createOrPatchObjectNeverAnnotation(modified client.Object, cl client.Client) error { + c := modified.DeepCopyObject().(client.Object) + found, err := getter.GetResourceInCache(modified.GetNamespace(), modified.GetName(), c, cl) + if err != nil { + return err + } else if !found { + return createObjectNeverAnnotation(modified, cl) + } + + gvk := modified.GetObjectKind().GroupVersionKind() + current, err := util.SetGroupVersionKind(c, &gvk) + if err != nil { + return err + } + + patchBytes, patchType, err := patcher.GeneratePatchBytes(current, modified) + if err != nil { + return err + } + + if len(patchBytes) == 0 || patchType == "" || string(patchBytes) == "{}" { + return nil + } + + return patchObjectWithType(current.(client.Object), patchBytes, patchType, cl) +} + func patchObjectWithType(obj client.Object, patchBytes []byte, patchType types.PatchType, cl client.Client) error { return cl.Patch(context.TODO(), obj, client.RawPatch(patchType, patchBytes)) } diff --git a/pkg/tool/kube/updater/clusterrole.go b/pkg/tool/kube/updater/clusterrole.go index 3ba0bcd51011f31567b63a8734db76a7760c1eb4..84a9014446debf065737d6fd13625aba222f6c4e 100644 --- a/pkg/tool/kube/updater/clusterrole.go +++ b/pkg/tool/kube/updater/clusterrole.go @@ -17,11 +17,36 @@ limitations under the License. package updater import ( - rbacv1beta1 "k8s.io/api/rbac/v1beta1" + "context" + + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - "sigs.k8s.io/controller-runtime/pkg/client" + "k8s.io/client-go/kubernetes" + + "github.com/koderover/zadig/pkg/tool/kube/util" ) -func DeleteClusterRoles(selector labels.Selector, cl client.Client) error { - return deleteObjectsWithDefaultOptions("", selector, &rbacv1beta1.ClusterRole{}, cl) +func DeleteClusterRoles(selector labels.Selector, clientset *kubernetes.Clientset) error { + version, err := clientset.Discovery().ServerVersion() + if err != nil { + return err + } + + deletePolicy := metav1.DeletePropagationForeground + if kubeclient.VersionLessThan122(version) { + err = clientset.RbacV1beta1().ClusterRoles().DeleteCollection(context.TODO(), metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }, metav1.ListOptions{ + LabelSelector: selector.String(), + }) + } else { + err = clientset.RbacV1().ClusterRoles().DeleteCollection(context.TODO(), metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }, metav1.ListOptions{ + LabelSelector: selector.String(), + }) + } + + return util.IgnoreNotFoundError(err) } diff --git a/pkg/tool/kube/updater/configmap.go b/pkg/tool/kube/updater/configmap.go index 6a79a8dd84a390ee40f85552e3ba6bf505aaa895..cf570256e1e4e53c0e51650bc25c1a6697fcb9b2 100644 --- a/pkg/tool/kube/updater/configmap.go +++ b/pkg/tool/kube/updater/configmap.go @@ -17,19 +17,40 @@ limitations under the License. package updater import ( + "context" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/koderover/zadig/pkg/tool/kube/util" ) -func DeleteConfigMaps(ns string, selector labels.Selector, cl client.Client) error { - return deleteObjectsWithDefaultOptions(ns, selector, &corev1.ConfigMap{}, cl) +func DeleteConfigMaps(namespace string, selector labels.Selector, clientset *kubernetes.Clientset) error { + deletePolicy := metav1.DeletePropagationForeground + err := clientset.CoreV1().ConfigMaps(namespace).DeleteCollection( + context.TODO(), + metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }, + metav1.ListOptions{ + LabelSelector: selector.String(), + }, + ) + + return util.IgnoreNotFoundError(err) } -func UpdateConfigMap(cm *corev1.ConfigMap, cl client.Client) error { - return updateObject(cm, cl) +func UpdateConfigMap(namespace string, cm *corev1.ConfigMap, clientset *kubernetes.Clientset) error { + _, err := clientset.CoreV1().ConfigMaps(namespace).Update( + context.TODO(), + cm, + metav1.UpdateOptions{}, + ) + return err } func DeleteConfigMap(ns, name string, cl client.Client) error { @@ -42,7 +63,7 @@ func DeleteConfigMap(ns, name string, cl client.Client) error { } func CreateConfigMap(cm *corev1.ConfigMap, cl client.Client) error { - return createObject(cm, cl) + return createObjectNeverAnnotation(cm, cl) } func DeleteConfigMapsAndWait(ns string, selector labels.Selector, cl client.Client) error { diff --git a/pkg/tool/kube/updater/cronjob.go b/pkg/tool/kube/updater/cronjob.go index 2db5313e2c3c06d1df434276486e72f318ad628a..9705698f6a16fa75dc71278d0099d42342c0e220 100644 --- a/pkg/tool/kube/updater/cronjob.go +++ b/pkg/tool/kube/updater/cronjob.go @@ -17,13 +17,30 @@ limitations under the License. package updater import ( + "context" + batchv1beta1 "k8s.io/api/batch/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/koderover/zadig/pkg/tool/kube/util" ) -func DeleteCronJobs(ns string, selector labels.Selector, cl client.Client) error { - return deleteObjectsWithDefaultOptions(ns, selector, &batchv1beta1.CronJob{}, cl) +func DeleteCronJobs(namespace string, selector labels.Selector, clientset *kubernetes.Clientset) error { + deletePolicy := metav1.DeletePropagationForeground + err := clientset.BatchV1().CronJobs(namespace).DeleteCollection( + context.TODO(), + metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }, + metav1.ListOptions{ + LabelSelector: selector.String(), + }, + ) + + return util.IgnoreNotFoundError(err) } func CreateOrPatchCronJob(cj *batchv1beta1.CronJob, cl client.Client) error { diff --git a/pkg/tool/kube/updater/deployment.go b/pkg/tool/kube/updater/deployment.go index 220d3fe97b2a1608d4ff36258eb113a5c87300ba..922bc897d11932f44b75d6e2af6e8cafe4f8c75c 100644 --- a/pkg/tool/kube/updater/deployment.go +++ b/pkg/tool/kube/updater/deployment.go @@ -18,6 +18,7 @@ package updater import ( "bytes" + "context" "fmt" "text/template" "time" @@ -25,7 +26,10 @@ import ( appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/koderover/zadig/pkg/tool/kube/util" ) var restartPatchTemplate = template.Must(template.New("restart-patch-template").Parse(`{ @@ -65,8 +69,19 @@ func RestartDeployment(ns, name string, cl client.Client) error { return nil } -func DeleteDeployments(ns string, selector labels.Selector, cl client.Client) error { - return deleteObjectsWithDefaultOptions(ns, selector, &appsv1.Deployment{}, cl) +func DeleteDeployments(namespace string, selector labels.Selector, clientset *kubernetes.Clientset) error { + deletePolicy := metav1.DeletePropagationForeground + err := clientset.AppsV1().Deployments(namespace).DeleteCollection( + context.TODO(), + metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }, + metav1.ListOptions{ + LabelSelector: selector.String(), + }, + ) + + return util.IgnoreNotFoundError(err) } func UpdateDeploymentImage(ns, name, container, image string, cl client.Client) error { diff --git a/pkg/tool/kube/updater/ingress.go b/pkg/tool/kube/updater/ingress.go index 0df90bbd9139aae7256294f364dc3319be2c55f1..646a92a5b722899a1ae87407eacb4568a2ce5185 100644 --- a/pkg/tool/kube/updater/ingress.go +++ b/pkg/tool/kube/updater/ingress.go @@ -17,11 +17,75 @@ limitations under the License. package updater import ( - extensionsv1beta1 "k8s.io/api/extensions/v1beta1" + "context" + + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" + v1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - "sigs.k8s.io/controller-runtime/pkg/client" + "k8s.io/client-go/kubernetes" + + "github.com/koderover/zadig/pkg/tool/kube/util" ) -func DeleteIngresses(ns string, selector labels.Selector, cl client.Client) error { - return deleteObjectsWithDefaultOptions(ns, selector, &extensionsv1beta1.Ingress{}, cl) +func DeleteIngresses(namespace string, selector labels.Selector, clientset *kubernetes.Clientset) error { + version, err := clientset.Discovery().ServerVersion() + if err != nil { + return err + } + + deletePolicy := metav1.DeletePropagationForeground + + if kubeclient.VersionLessThan122(version) { + err = clientset.ExtensionsV1beta1().Ingresses(namespace).DeleteCollection( + context.TODO(), + metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }, metav1.ListOptions{ + LabelSelector: selector.String(), + }) + } else { + err = clientset.NetworkingV1().Ingresses(namespace).DeleteCollection( + context.TODO(), + metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }, + metav1.ListOptions{ + LabelSelector: selector.String(), + }, + ) + } + + return util.IgnoreNotFoundError(err) +} + +func DeleteIngresseWithName(namespace, name string, clientset *kubernetes.Clientset) error { + version, err := clientset.Discovery().ServerVersion() + if err != nil { + return err + } + + deletePolicy := metav1.DeletePropagationForeground + if kubeclient.VersionLessThan122(version) { + return clientset.ExtensionsV1beta1().Ingresses(namespace).Delete( + context.TODO(), name, + metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }) + } + + return clientset.NetworkingV1().Ingresses(namespace).Delete( + context.TODO(), name, + metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }, + ) +} + +func CreateIngress(namespace string, ingress *v1.Ingress, clientset *kubernetes.Clientset) error { + _, err := clientset.NetworkingV1().Ingresses(namespace).Create( + context.TODO(), ingress, + metav1.CreateOptions{}, + ) + return err } diff --git a/pkg/tool/kube/updater/job.go b/pkg/tool/kube/updater/job.go index e3fc25424b88feb64074887e62d5ad20a1c3d94d..1ad6612f3cb64526e30682323a0f4d66aca5dde5 100644 --- a/pkg/tool/kube/updater/job.go +++ b/pkg/tool/kube/updater/job.go @@ -17,15 +17,31 @@ limitations under the License. package updater import ( + "context" + batchv1 "k8s.io/api/batch/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/koderover/zadig/pkg/tool/kube/util" ) -func DeleteJobs(ns string, selector labels.Selector, cl client.Client) error { - return deleteObjectsWithDefaultOptions(ns, selector, &batchv1.Job{}, cl) +func DeleteJobs(namespace string, selector labels.Selector, clientset *kubernetes.Clientset) error { + deletePolicy := metav1.DeletePropagationForeground + err := clientset.BatchV1().Jobs(namespace).DeleteCollection( + context.TODO(), + metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }, + metav1.ListOptions{ + LabelSelector: selector.String(), + }, + ) + + return util.IgnoreNotFoundError(err) } func CreateJob(job *batchv1.Job, cl client.Client) error { diff --git a/pkg/tool/kube/updater/namespace.go b/pkg/tool/kube/updater/namespace.go index d6c7a540f55db8d6749a1000673e10a5994d2a3a..1d2f20d2544023c43f902c1a358796c32995e7dd 100644 --- a/pkg/tool/kube/updater/namespace.go +++ b/pkg/tool/kube/updater/namespace.go @@ -17,36 +17,19 @@ limitations under the License. package updater import ( + "context" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/koderover/zadig/pkg/tool/kube/getter" ) -func DeleteMatchingNamespace(ns string, selector labels.Selector, cl client.Client) error { - namespace, found, err := getter.GetNamespace(ns, cl) - if err != nil { - return err - } else if !found { - return nil - } - - if !selector.Matches(labels.Set(namespace.Labels)) { - return nil - } - - return DeleteNamespace(ns, cl) -} - -func DeleteNamespace(ns string, cl client.Client) error { - return deleteObjectWithDefaultOptions(&corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "", - Name: ns, - }, - }, cl) +func DeleteNamespace(name string, clientset *kubernetes.Clientset) error { + deletePolicy := metav1.DeletePropagationForeground + return clientset.CoreV1().Namespaces().Delete(context.TODO(), name, metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }) } func CreateNamespace(ns *corev1.Namespace, cl client.Client) error { diff --git a/pkg/tool/kube/updater/pod.go b/pkg/tool/kube/updater/pod.go index 3e509b2b7e4e5a4db2b98cbed2c48da2c1cf8c40..8b480f3f470792903c179439a77fa5177fcef231 100644 --- a/pkg/tool/kube/updater/pod.go +++ b/pkg/tool/kube/updater/pod.go @@ -17,14 +17,30 @@ limitations under the License. package updater import ( + "context" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/koderover/zadig/pkg/tool/kube/util" ) -func DeletePods(ns string, selector labels.Selector, cl client.Client) error { - return deleteObjectsWithDefaultOptions(ns, selector, &corev1.Pod{}, cl) +func DeletePods(namespace string, selector labels.Selector, clientset *kubernetes.Clientset) error { + deletePolicy := metav1.DeletePropagationForeground + err := clientset.CoreV1().Pods(namespace).DeleteCollection( + context.TODO(), + metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }, + metav1.ListOptions{ + LabelSelector: selector.String(), + }, + ) + + return util.IgnoreNotFoundError(err) } func DeletePod(ns, name string, cl client.Client) error { diff --git a/pkg/tool/kube/updater/pv.go b/pkg/tool/kube/updater/pv.go index ef3fd4e05970c80172dcad75f9de814a83f99115..e9f890fc4098839d612d88c0fee1d10e5b922b9a 100644 --- a/pkg/tool/kube/updater/pv.go +++ b/pkg/tool/kube/updater/pv.go @@ -17,11 +17,26 @@ limitations under the License. package updater import ( - corev1 "k8s.io/api/core/v1" + "context" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - "sigs.k8s.io/controller-runtime/pkg/client" + "k8s.io/client-go/kubernetes" + + "github.com/koderover/zadig/pkg/tool/kube/util" ) -func DeletePersistentVolumes(ns string, selector labels.Selector, cl client.Client) error { - return deleteObjectsWithDefaultOptions(ns, selector, &corev1.PersistentVolume{}, cl) +func DeletePersistentVolumes(selector labels.Selector, clientset *kubernetes.Clientset) error { + deletePolicy := metav1.DeletePropagationForeground + err := clientset.CoreV1().PersistentVolumes().DeleteCollection( + context.TODO(), + metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }, + metav1.ListOptions{ + LabelSelector: selector.String(), + }, + ) + + return util.IgnoreNotFoundError(err) } diff --git a/pkg/tool/kube/updater/pvc.go b/pkg/tool/kube/updater/pvc.go index 2117dacecb9ba2d9491dc47e7a32167996c1bf20..f28e765d681b29219047b1e320cb4b5a389b5d14 100644 --- a/pkg/tool/kube/updater/pvc.go +++ b/pkg/tool/kube/updater/pvc.go @@ -17,11 +17,49 @@ limitations under the License. package updater import ( + "context" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - "sigs.k8s.io/controller-runtime/pkg/client" + "k8s.io/client-go/kubernetes" + + "github.com/koderover/zadig/pkg/tool/kube/util" ) -func DeletePersistentVolumeClaims(ns string, selector labels.Selector, cl client.Client) error { - return deleteObjectsWithDefaultOptions(ns, selector, &corev1.PersistentVolumeClaim{}, cl) +func DeletePersistentVolumeClaims(namespace string, selector labels.Selector, clientset *kubernetes.Clientset) error { + deletePolicy := metav1.DeletePropagationForeground + err := clientset.CoreV1().PersistentVolumeClaims(namespace).DeleteCollection( + context.TODO(), + metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }, + metav1.ListOptions{ + LabelSelector: selector.String(), + }, + ) + + return util.IgnoreNotFoundError(err) +} + +func DeletePvcWithName(namespace, name string, clientset *kubernetes.Clientset) error { + deletePolicy := metav1.DeletePropagationForeground + return clientset.CoreV1().PersistentVolumeClaims(namespace).Delete( + context.TODO(), name, + metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }, + ) +} + +func CreatePvc(namespace string, pvc *corev1.PersistentVolumeClaim, clientset *kubernetes.Clientset) error { + _, err := clientset.CoreV1().PersistentVolumeClaims(namespace).Create(context.TODO(), pvc, + metav1.CreateOptions{}) + return err +} + +func UpdatePvc(namespace string, pvc *corev1.PersistentVolumeClaim, clientset *kubernetes.Clientset) error { + _, err := clientset.CoreV1().PersistentVolumeClaims(namespace).Update(context.TODO(), pvc, + metav1.UpdateOptions{}) + return err } diff --git a/pkg/tool/kube/updater/replicaset.go b/pkg/tool/kube/updater/replicaset.go index 525931050d70e0ee2c9a965a20c24110d968ad0c..ff546ccfd6178c785bfadf36d8c4b4703c0a27ad 100644 --- a/pkg/tool/kube/updater/replicaset.go +++ b/pkg/tool/kube/updater/replicaset.go @@ -17,11 +17,26 @@ limitations under the License. package updater import ( - appsv1 "k8s.io/api/apps/v1" + "context" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - "sigs.k8s.io/controller-runtime/pkg/client" + "k8s.io/client-go/kubernetes" + + "github.com/koderover/zadig/pkg/tool/kube/util" ) -func DeleteReplicaSets(ns string, selector labels.Selector, cl client.Client) error { - return deleteObjectsWithDefaultOptions(ns, selector, &appsv1.ReplicaSet{}, cl) +func DeleteReplicaSets(namespace string, selector labels.Selector, clientset *kubernetes.Clientset) error { + deletePolicy := metav1.DeletePropagationForeground + err := clientset.AppsV1().ReplicaSets(namespace).DeleteCollection( + context.TODO(), + metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }, + metav1.ListOptions{ + LabelSelector: selector.String(), + }, + ) + + return util.IgnoreNotFoundError(err) } diff --git a/pkg/tool/kube/updater/role.go b/pkg/tool/kube/updater/role.go index c6ac3bb53024883ec9c1385c30932953b99b6425..6c0511c46fcccee66ab6ff6bb2c35a9b88c2f5a4 100644 --- a/pkg/tool/kube/updater/role.go +++ b/pkg/tool/kube/updater/role.go @@ -17,19 +17,43 @@ limitations under the License. package updater import ( - rbacv1beta1 "k8s.io/api/rbac/v1beta1" - "k8s.io/apimachinery/pkg/labels" - "sigs.k8s.io/controller-runtime/pkg/client" -) + "context" -func DeleteRoles(ns string, selector labels.Selector, cl client.Client) error { - return deleteObjectsWithDefaultOptions(ns, selector, &rbacv1beta1.Role{}, cl) -} + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" -func CreateRole(role *rbacv1beta1.Role, cl client.Client) error { - return createObject(role, cl) -} + "github.com/koderover/zadig/pkg/tool/kube/util" +) -func UpdateRole(role *rbacv1beta1.Role, cl client.Client) error { - return updateObject(role, cl) +func DeleteRoles(namespace string, selector labels.Selector, clientset *kubernetes.Clientset) error { + version, err := clientset.Discovery().ServerVersion() + if err != nil { + return err + } + + deletePolicy := metav1.DeletePropagationForeground + if kubeclient.VersionLessThan122(version) { + err = clientset.RbacV1beta1().Roles(namespace).DeleteCollection( + context.TODO(), + metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }, metav1.ListOptions{ + LabelSelector: selector.String(), + }, + ) + } else { + err = clientset.RbacV1().Roles(namespace).DeleteCollection( + context.TODO(), + metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }, + metav1.ListOptions{ + LabelSelector: selector.String(), + }, + ) + } + + return util.IgnoreNotFoundError(err) } diff --git a/pkg/tool/kube/updater/rolebinding.go b/pkg/tool/kube/updater/rolebinding.go index 293fa00cece30964ab8629823493935b275f83ce..cc9cceb9cf9ecbe0d7673ba75987ed2b3c3517e0 100644 --- a/pkg/tool/kube/updater/rolebinding.go +++ b/pkg/tool/kube/updater/rolebinding.go @@ -17,19 +17,26 @@ limitations under the License. package updater import ( - rbacv1beta1 "k8s.io/api/rbac/v1beta1" - "k8s.io/apimachinery/pkg/labels" - "sigs.k8s.io/controller-runtime/pkg/client" -) + "context" -func DeleteRoleBindings(ns string, selector labels.Selector, cl client.Client) error { - return deleteObjectsWithDefaultOptions(ns, selector, &rbacv1beta1.RoleBinding{}, cl) -} + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" -func CreateRoleBinding(rb *rbacv1beta1.RoleBinding, cl client.Client) error { - return createObject(rb, cl) -} + "github.com/koderover/zadig/pkg/tool/kube/util" +) -func UpdateRoleBinding(rb *rbacv1beta1.RoleBinding, cl client.Client) error { - return updateObject(rb, cl) +func DeleteRoleBindings(namespace string, selector labels.Selector, clientset *kubernetes.Clientset) error { + deletePolicy := metav1.DeletePropagationForeground + err := clientset.RbacV1().RoleBindings(namespace).DeleteCollection( + context.TODO(), + metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }, + metav1.ListOptions{ + LabelSelector: selector.String(), + }, + ) + + return util.IgnoreNotFoundError(err) } diff --git a/pkg/tool/kube/updater/secret.go b/pkg/tool/kube/updater/secret.go index 731998f0c56855a69084c306760c559027ef46d5..5a541121771bab1c20ac92912c0b2bf604e0200d 100644 --- a/pkg/tool/kube/updater/secret.go +++ b/pkg/tool/kube/updater/secret.go @@ -17,15 +17,42 @@ limitations under the License. package updater import ( + "context" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/koderover/zadig/pkg/tool/kube/util" ) -func DeleteSecrets(ns string, selector labels.Selector, cl client.Client) error { - return deleteObjectsWithDefaultOptions(ns, selector, &corev1.Secret{}, cl) +func DeleteSecrets(namespace string, selector labels.Selector, clientset *kubernetes.Clientset) error { + deletePolicy := metav1.DeletePropagationForeground + err := clientset.CoreV1().Secrets(namespace).DeleteCollection( + context.TODO(), + metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }, + metav1.ListOptions{ + LabelSelector: selector.String(), + }, + ) + + return util.IgnoreNotFoundError(err) } func UpdateOrCreateSecret(s *corev1.Secret, cl client.Client) error { return updateOrCreateObject(s, cl) } + +func DeleteSecretWithName(ns, name string, cl client.Client) error { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: name, + }, + } + return deleteObject(secret, cl) +} diff --git a/pkg/tool/kube/updater/service.go b/pkg/tool/kube/updater/service.go index ad2192ac6167100d4beb7b3a73d4e38a88812f45..270b74aebfaf1974d500b1efd0aa877fdb4070ea 100644 --- a/pkg/tool/kube/updater/service.go +++ b/pkg/tool/kube/updater/service.go @@ -17,39 +17,33 @@ limitations under the License. package updater import ( - corev1 "k8s.io/api/core/v1" + "context" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/koderover/zadig/pkg/tool/kube/getter" + "k8s.io/client-go/kubernetes" ) -// service does not support deleteCollection -// see here for details: https://github.com/kubernetes/kubernetes/issues/68468#issuecomment-419981870 -func DeleteServices(ns string, selector labels.Selector, cl client.Client) error { - services, err := getter.ListServices(ns, selector, cl) +func DeleteServices(namespace string, selector labels.Selector, clientset *kubernetes.Clientset) error { + services, err := clientset.CoreV1().Services(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: selector.String()}) if err != nil { return err } var lastErr error - for _, service := range services { - err := DeleteService(ns, service.Name, cl) + deletePolicy := metav1.DeletePropagationForeground + for _, svc := range services.Items { + err := clientset.CoreV1().Services(namespace).Delete( + context.TODO(), + svc.Name, + metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }, + ) if err != nil { - lastErr = err } + lastErr = err } return lastErr } - -func DeleteService(ns, name string, cl client.Client) error { - deletePolicy := metav1.DeletePropagationForeground - return deleteObject(&corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: ns, - Name: name, - }, - }, cl, &client.DeleteOptions{PropagationPolicy: &deletePolicy}) -} diff --git a/pkg/tool/kube/updater/serviceaccount.go b/pkg/tool/kube/updater/serviceaccount.go index 020011ff9b13ee696fe93e9cff4cc20c6775e08a..23e1fc23ddbb02c22b881869687bdf68ae07675e 100644 --- a/pkg/tool/kube/updater/serviceaccount.go +++ b/pkg/tool/kube/updater/serviceaccount.go @@ -17,13 +17,30 @@ limitations under the License. package updater import ( + "context" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/koderover/zadig/pkg/tool/kube/util" ) -func DeleteServiceAccounts(ns string, selector labels.Selector, cl client.Client) error { - return deleteObjectsWithDefaultOptions(ns, selector, &corev1.ServiceAccount{}, cl) +func DeleteServiceAccounts(namespace string, selector labels.Selector, clientset *kubernetes.Clientset) error { + deletePolicy := metav1.DeletePropagationForeground + err := clientset.CoreV1().ServiceAccounts(namespace).DeleteCollection( + context.TODO(), + metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }, + metav1.ListOptions{ + LabelSelector: selector.String(), + }, + ) + + return util.IgnoreNotFoundError(err) } func CreateServiceAccount(sa *corev1.ServiceAccount, cl client.Client) error { diff --git a/pkg/tool/kube/updater/statefulset.go b/pkg/tool/kube/updater/statefulset.go index 0f288df42aeed68bc1831f32422382ca135463c7..c093245f5cd7e7916000fc70fbc4820c8b0d2b7f 100644 --- a/pkg/tool/kube/updater/statefulset.go +++ b/pkg/tool/kube/updater/statefulset.go @@ -18,10 +18,12 @@ package updater import ( "bytes" + "context" "fmt" "time" "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -53,8 +55,17 @@ func RestartStatefulSet(ns, name string, cl client.Client) error { return nil } -func DeleteStatefulSets(ns string, selector labels.Selector, cl client.Client) error { - return deleteObjectsWithDefaultOptions(ns, selector, &appsv1.StatefulSet{}, cl) +func DeleteStatefulSets(namespace string, selector labels.Selector, clientset *kubernetes.Clientset) error { + deletePolicy := metav1.DeletePropagationForeground + return clientset.AppsV1().StatefulSets(namespace).DeleteCollection( + context.TODO(), + metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }, + metav1.ListOptions{ + LabelSelector: selector.String(), + }, + ) } func UpdateStatefulSetImage(ns, name, container, image string, cl client.Client) error { diff --git a/pkg/tool/kube/updater/unstructured.go b/pkg/tool/kube/updater/unstructured.go index d3fc2ef522554fdfa18890ee08a32ad332c51661..a9421b1a305b11dd2ae2d6925515732c82d40ae8 100644 --- a/pkg/tool/kube/updater/unstructured.go +++ b/pkg/tool/kube/updater/unstructured.go @@ -25,6 +25,14 @@ func CreateOrPatchUnstructured(u *unstructured.Unstructured, cl client.Client) e return createOrPatchObject(u, cl) } +func CreateOrPatchUnstructuredNeverAnnotation(u *unstructured.Unstructured, cl client.Client) error { + return createOrPatchObjectNeverAnnotation(u, cl) +} + +func UpdateOrCreateUnstructured(u *unstructured.Unstructured, cl client.Client) error { + return updateOrCreateObject(u, cl) +} + func DeleteUnstructured(u *unstructured.Unstructured, cl client.Client) error { return deleteObjectWithDefaultOptions(u, cl) } diff --git a/pkg/tool/kube/util/utils.go b/pkg/tool/kube/util/utils.go new file mode 100644 index 0000000000000000000000000000000000000000..3a9c073215437e434a0c41f9764bd509432facae --- /dev/null +++ b/pkg/tool/kube/util/utils.go @@ -0,0 +1,96 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "bytes" + "fmt" + "io" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer/yaml" + yamlutil "k8s.io/apimachinery/pkg/util/yaml" +) + +type ObjGVK struct { + Object runtime.Object + GVK *schema.GroupVersionKind +} + +func IgnoreNotFoundError(err error) error { + if apierrors.IsNotFound(err) { + return nil + } + + return err +} + +func ParseManifest(manifest string) ([]ObjGVK, error) { + decoder := yamlutil.NewYAMLOrJSONDecoder(bytes.NewReader([]byte(manifest)), 5*1024*1024) + + res := []ObjGVK{} + for { + var rawObj runtime.RawExtension + + err := decoder.Decode(&rawObj) + if err == io.EOF { + break + } + if err != nil { + return res, fmt.Errorf("failed to decode raw extension: %s", err) + } + + obj, gvk, err := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme).Decode(rawObj.Raw, nil, nil) + if err != nil { + return res, fmt.Errorf("failed to decode RawExtension to runtime.Object and GVK: %s", err) + } + + res = append(res, ObjGVK{ + Object: obj, + GVK: gvk, + }) + } + + return res, nil +} + +func GetSvcNamesFromManifest(manifest string) ([]string, error) { + objgvks, err := ParseManifest(manifest) + if err != nil { + return nil, err + } + + svcNames := []string{} + for _, objgvk := range objgvks { + if objgvk.GVK.GroupKind().String() != "Service" { + continue + } + + objMeta, err := meta.Accessor(objgvk.Object) + if err != nil { + return nil, fmt.Errorf("failed to construct object meta: %s", err) + } + + svcNames = append(svcNames, objMeta.GetName()) + } + + return svcNames, nil +} diff --git a/pkg/tool/mail/mail.go b/pkg/tool/mail/mail.go new file mode 100644 index 0000000000000000000000000000000000000000..f1faf30d126b89fe170c031738b43d6d8cdd192b --- /dev/null +++ b/pkg/tool/mail/mail.go @@ -0,0 +1,66 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mail + +import ( + "bytes" + "html/template" + + "gopkg.in/gomail.v2" +) + +type EmailParams struct { + From string + To string + Subject string + Body string + Host string + Port int + UserName string + Password string +} + +func SendEmail(param *EmailParams) error { + m := gomail.NewMessage() + m.SetHeader("From", param.From) + m.SetHeader("To", param.To) + m.SetHeader("Subject", param.Subject) + m.SetBody("text/html", param.Body) + + d := gomail.NewDialer(param.Host, param.Port, param.UserName, param.Password) + + if err := d.DialAndSend(m); err != nil { + return err + } + return nil +} + +type GenerateUrl struct { + Url string +} + +func RenderEmailTemplate(url string, html string) (content string, err error) { + buf := new(bytes.Buffer) + t := template.Must(template.New("email").Funcs(template.FuncMap{}).Parse(html)) + + if err = t.Execute(buf, &GenerateUrl{ + Url: url, + }); err != nil { + return "", err + } + return buf.String(), nil +} diff --git a/pkg/tool/opa/bundle.go b/pkg/tool/opa/bundle.go new file mode 100644 index 0000000000000000000000000000000000000000..b04c8c045dee42d3298adcf8ed20f1f067b9b8df --- /dev/null +++ b/pkg/tool/opa/bundle.go @@ -0,0 +1,165 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package opa + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "path/filepath" + + "github.com/27149chen/afero" + + "github.com/koderover/zadig/pkg/tool/log" + fsutil "github.com/koderover/zadig/pkg/util/fs" +) + +const ( + manifestPath = ".manifest" +) + +type Manifest struct { + Revision string `json:"revision"` + Roots []string `json:"roots"` +} + +type DataSpec struct { + Data interface{} + Path string +} + +type serializedDataSpec struct { + Content []byte + Path string +} + +type Bundle struct { + Data []*DataSpec + Roots []string + hash string + serialized []*serializedDataSpec +} + +func (b *Bundle) Rehash() (string, error) { + if !b.validate() { + return "", fmt.Errorf("validation failed, data or roots is missing") + } + + var allContents []byte + var err error + b.serialized = nil + + for _, file := range b.Data { + var content []byte + switch c := file.Data.(type) { + case []byte: + content = c + default: + content, err = json.MarshalIndent(c, "", " ") + if err != nil { + log.Errorf("Failed to marshal file %s, err: %s", file.Path, err) + return "", err + } + } + + b.serialized = append(b.serialized, &serializedDataSpec{ + Content: content, + Path: file.Path, + }) + allContents = append(allContents, content...) + } + + h := sha256.New() + h.Write(allContents) + b.hash = fmt.Sprintf("%x", h.Sum(nil)) + + return b.hash, nil +} + +func (b *Bundle) Hash() (string, error) { + if b.hash != "" { + return b.hash, nil + } + + return b.Rehash() +} + +func (b *Bundle) Save(parentPath string) error { + mf, err := b.serializeOPAManifest() + if err != nil { + log.Errorf("Failed to serialize OPA manifest, err: %s", err) + return err + } + b.serialized = append(b.serialized, mf) + + cacheFS := afero.NewMemMapFs() + for _, file := range b.serialized { + err = cacheFS.MkdirAll(filepath.Dir(file.Path), 0755) + if err != nil { + log.Errorf("Failed to create path %s, err: %s", filepath.Dir(file.Path), err) + return err + } + err = afero.WriteFile(cacheFS, file.Path, file.Content, 0644) + if err != nil { + log.Errorf("Failed to write file %s, err: %s", file.Path, err) + return err + } + } + + tarball := "bundle.tar.gz" + path := filepath.Join(parentPath, tarball) + if err = fsutil.Tar(afero.NewIOFS(cacheFS), path); err != nil { + log.Errorf("Failed to archive tarball %s, err: %s", path, err) + return err + } + + return nil +} + +func (b *Bundle) generateOPAManifest() (*Manifest, error) { + revision, err := b.Hash() + if err != nil { + log.Errorf("Failed to get hash, err: %s", err) + return nil, err + } + + return &Manifest{ + Revision: revision, + Roots: b.Roots, + }, nil +} + +func (b *Bundle) serializeOPAManifest() (*serializedDataSpec, error) { + mf, err := b.generateOPAManifest() + if err != nil { + return nil, err + } + + content, err := json.MarshalIndent(mf, "", " ") + if err != nil { + return nil, err + } + + return &serializedDataSpec{ + Content: content, + Path: manifestPath, + }, nil +} + +func (b *Bundle) validate() bool { + return len(b.Data) > 0 && len(b.Roots) > 0 +} diff --git a/pkg/tool/registries/dind.go b/pkg/tool/registries/dind.go new file mode 100644 index 0000000000000000000000000000000000000000..4fd001c2b34e8fca607e1d4b32dfacccc5fba84f --- /dev/null +++ b/pkg/tool/registries/dind.go @@ -0,0 +1,183 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registries + +import ( + "context" + "encoding/base64" + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "strings" + + "github.com/koderover/zadig/pkg/tool/log" +) + +func PrepareDinD(dynamicClient dynamic.Interface, namespace string, regList []*RegistryInfoForDinDUpdate) error { + // set statefulset GVR + stsResource := schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "statefulsets"} + + volumeMountList := make([]interface{}, 0) + volumeList := make([]interface{}, 0) + insecureRegistryList := make([]interface{}, 0) + + mountFlag := false + insecureFlag := false + sourceList := make([]interface{}, 0) + + for _, reg := range regList { + // compatibility changes before 1.11 + if reg.AdvancedSetting != nil { + // remove the http:// https:// prefix + addr := strings.Split(reg.RegAddr, "//") + // if a registry is marked as insecure, we add a record to insecure-registries + if !reg.AdvancedSetting.TLSEnabled { + insecureFlag = true + insecureRegistryList = append(insecureRegistryList, fmt.Sprintf("--insecure-registry=%s", addr[1])) + } + // if a registry is marked as secure and a TLS cert is given, we mount this certificate to dind daemon + if reg.AdvancedSetting.TLSEnabled && reg.AdvancedSetting.TLSCert != "" { + mountName := fmt.Sprintf("%s-cert", reg.ID.Hex()) + err := ensureCertificateSecret(dynamicClient, mountName, namespace, reg.AdvancedSetting.TLSCert) + if err != nil { + log.Errorf("failed to ensure secret: %s, the error is: %s", mountName, err) + return err + } + // create secret mount info + secretMount := map[string]interface{}{ + "name": mountName, + "items": []interface{}{ + map[string]interface{}{ + "key": "ca.crt", + "path": fmt.Sprintf("%s/%s", addr[1], "ca.crt"), + }, + }, + } + + // create projected volumes sources list + sourceList = append(sourceList, map[string]interface{}{ + "secret": secretMount, + }) + + // set mountFlag to add mounting volume to dind + mountFlag = true + } + } + } + + if mountFlag { + volumeMountMap := map[string]interface{}{ + "mountPath": "/etc/docker/certs.d", + "name": "cert-mount", + } + volumeMountList = append(volumeMountList, volumeMountMap) + volumeList = append(volumeList, map[string]interface{}{ + "name": "cert-mount", + "projected": map[string]interface{}{ + "sources": sourceList, + }, + }) + } + + result, getErr := dynamicClient.Resource(stsResource).Namespace(namespace).Get(context.TODO(), "dind", metav1.GetOptions{}) + if getErr != nil { + log.Errorf("failed to get dind statefulset, the error is: %s", getErr) + return getErr + } + + // extract spec containers + containers, found, err := unstructured.NestedSlice(result.Object, "spec", "template", "spec", "containers") + if err != nil || !found || containers == nil { + return err + } + + if mountFlag { + // update spec.template.spec.containers[0].volumeMounts + if err := unstructured.SetNestedField(containers[0].(map[string]interface{}), volumeMountList, "volumeMounts"); err != nil { + return err + } + // update spec.template.spec.volumes + if err := unstructured.SetNestedField(result.Object, volumeList, "spec", "template", "spec", "volumes"); err != nil { + return err + } + } + + if insecureFlag { + // update spec.template.spec.containers[0].args + if err := unstructured.SetNestedField(containers[0].(map[string]interface{}), insecureRegistryList, "args"); err != nil { + return err + } + } + + if err := unstructured.SetNestedField(result.Object, containers, "spec", "template", "spec", "containers"); err != nil { + return err + } + + _, updateErr := dynamicClient.Resource(stsResource).Namespace(namespace).Update(context.TODO(), result, metav1.UpdateOptions{}) + if updateErr != nil { + log.Errorf("failed to update dind, the error is: %s", updateErr) + } + return updateErr +} + +func ensureCertificateSecret(dynamicClient dynamic.Interface, secretName, namespace, cert string) error { + certificateString := base64.StdEncoding.EncodeToString([]byte(cert)) + datamap := map[string]interface{}{ + "ca.crt": certificateString, + } + + // setup secret GVR for future use + secretsGVR := schema.GroupVersionResource{ + Version: "v1", + Resource: "secrets", + } + + secret, err := dynamicClient.Resource(secretsGVR).Namespace(namespace).Get(context.TODO(), secretName, metav1.GetOptions{}) + // if there is an error, either because of not found or anything else, we try to create a secret with the given information + if err != nil { + secret := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Secret", + "metadata": map[string]interface{}{ + "name": secretName, + }, + "type": "Opaque", + "data": datamap, + }, + } + + _, err := dynamicClient.Resource(secretsGVR).Namespace(namespace).Create(context.TODO(), secret, metav1.CreateOptions{}) + if err != nil { + log.Errorf("failed to create secret: %s, the error is: %s", secretName, err) + } + return err + } else { + if err := unstructured.SetNestedField(secret.Object, datamap, "data"); err != nil { + log.Errorf("failed to set data in secret object, the error is: %s", err) + return err + } + _, err := dynamicClient.Resource(secretsGVR).Namespace(namespace).Update(context.TODO(), secret, metav1.UpdateOptions{}) + if err != nil { + log.Errorf("failed to update secret: %s, the error is: %s", secretName, err) + } + return err + } +} diff --git a/pkg/tool/registries/types.go b/pkg/tool/registries/types.go new file mode 100644 index 0000000000000000000000000000000000000000..53908947f8fdc3d596926d5a4c5e468c18bbc8be --- /dev/null +++ b/pkg/tool/registries/types.go @@ -0,0 +1,31 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registries + +import "go.mongodb.org/mongo-driver/bson/primitive" + +// RegistryInfoForDinDUpdate is the structure used to update dind, several unused field is omitted to avoid complication +type RegistryInfoForDinDUpdate struct { + ID primitive.ObjectID + RegAddr string + AdvancedSetting *RegistryAdvancedSetting +} + +type RegistryAdvancedSetting struct { + TLSEnabled bool + TLSCert string +} diff --git a/pkg/tool/remotedialer/client.go b/pkg/tool/remotedialer/client.go index fdfe85c6b0844b9a1fc5f47c9ef1a750f84eadd0..32981a7c8b3769051b29ef0803e9aefdc7c2628e 100644 --- a/pkg/tool/remotedialer/client.go +++ b/pkg/tool/remotedialer/client.go @@ -18,6 +18,7 @@ package remotedialer import ( "context" + "io/ioutil" "net/http" "time" @@ -25,39 +26,69 @@ import ( "github.com/sirupsen/logrus" ) +// ConnectAuthorizer custom for authorization type ConnectAuthorizer func(proto, address string) bool -func ClientConnect(wsURL string, headers http.Header, dialer *websocket.Dialer, auth ConnectAuthorizer, onConnect func(context.Context) error) { - if err := connectToProxy(wsURL, headers, auth, dialer, onConnect); err != nil { - logrus.WithError(err).Error("Failed to connect to proxy") +// ClientConnect connect to WS and wait 5 seconds when error +func ClientConnect(ctx context.Context, wsURL string, headers http.Header, dialer *websocket.Dialer, + auth ConnectAuthorizer, onConnect func(context.Context, *Session) error) error { + if err := ConnectToProxy(ctx, wsURL, headers, auth, dialer, onConnect); err != nil { + logrus.WithError(err).Error("Remotedialer proxy error") time.Sleep(time.Duration(5) * time.Second) + return err } + return nil } -func connectToProxy(proxyURL string, headers http.Header, auth ConnectAuthorizer, dialer *websocket.Dialer, onConnect func(context.Context) error) error { +// ConnectToProxy connect to websocket server +func ConnectToProxy(rootCtx context.Context, proxyURL string, headers http.Header, auth ConnectAuthorizer, dialer *websocket.Dialer, onConnect func(context.Context, *Session) error) error { logrus.WithField("url", proxyURL).Info("Connecting to proxy") if dialer == nil { - dialer = &websocket.Dialer{HandshakeTimeout: 10 * time.Second} + dialer = &websocket.Dialer{Proxy: http.ProxyFromEnvironment, HandshakeTimeout: HandshakeTimeOut} } - ws, _, err := dialer.Dial(proxyURL, headers) + ws, resp, err := dialer.DialContext(rootCtx, proxyURL, headers) if err != nil { - logrus.WithError(err).Error("Failed to connect to proxy") + if resp == nil { + logrus.WithError(err).Errorf("Failed to connect to proxy. Empty dialer response") + } else { + rb, err2 := ioutil.ReadAll(resp.Body) + if err2 != nil { + logrus.WithError(err).Errorf("Failed to connect to proxy. Response status: %v - %v. Couldn't read response body (err: %v)", resp.StatusCode, resp.Status, err2) + } else { + logrus.WithError(err).Errorf("Failed to connect to proxy. Response status: %v - %v. Response body: %s", resp.StatusCode, resp.Status, rb) + } + } return err } defer ws.Close() - ctx, cancel := context.WithCancel(context.Background()) + result := make(chan error, 2) + + ctx, cancel := context.WithCancel(rootCtx) defer cancel() + session := NewClientSession(auth, ws) + defer session.Close() + if onConnect != nil { - if err := onConnect(ctx); err != nil { - return err - } + go func() { + if err := onConnect(ctx, session); err != nil { + result <- err + } + }() } - session := NewClientSession(auth, ws) - _, err = session.Serve() - session.Close() - return err + go func() { + _, err = session.Serve(ctx) + result <- err + }() + + select { + case <-ctx.Done(): + logrus.WithField("url", proxyURL).WithField("err", ctx.Err()).Info("Proxy done") + return nil + case err := <-result: + return err + } } diff --git a/pkg/tool/remotedialer/client_dialer.go b/pkg/tool/remotedialer/client_dialer.go index 2bb5401ed1f45e8801792667e95e93d7b58b3098..5fd49b4f1957a5aa7190cb2598990c95245c8891 100644 --- a/pkg/tool/remotedialer/client_dialer.go +++ b/pkg/tool/remotedialer/client_dialer.go @@ -17,13 +17,14 @@ limitations under the License. package remotedialer import ( + "context" "io" "net" "sync" "time" ) -func clientDial(dialer Dialer, conn *connection, message *message) { +func clientDial(ctx context.Context, dialer Dialer, conn *connection, message *message) { defer conn.Close() var ( @@ -31,15 +32,14 @@ func clientDial(dialer Dialer, conn *connection, message *message) { err error ) + ctx, cancel := context.WithDeadline(ctx, time.Now().Add(time.Minute)) if dialer == nil { - netDialer := &net.Dialer{ - Timeout: time.Duration(message.deadline) * time.Millisecond, - KeepAlive: 30 * time.Second, - } - netConn, err = netDialer.Dial(message.proto, message.address) + d := net.Dialer{} + netConn, err = d.DialContext(ctx, message.proto, message.address) } else { - netConn, err = dialer(message.proto, message.address) + netConn, err = dialer(ctx, message.proto, message.address) } + cancel() if err != nil { conn.tunnelClose(err) diff --git a/pkg/tool/remotedialer/connection.go b/pkg/tool/remotedialer/connection.go index 6e8870aaae491cba8772ad4c52abbc2c10cbbe63..3538fa0b9be33487386d14c1d53714dfb500ca5d 100644 --- a/pkg/tool/remotedialer/connection.go +++ b/pkg/tool/remotedialer/connection.go @@ -17,20 +17,15 @@ limitations under the License. package remotedialer import ( - "errors" "io" "net" - "sync" "time" ) type connection struct { - sync.Mutex - err error writeDeadline time.Time - buf chan []byte - readBuf []byte + buffer *readBuffer addr addr session *Session connID int64 @@ -38,13 +33,13 @@ type connection struct { func newConnection(connID int64, session *Session, proto, address string) *connection { c := &connection{ + buffer: newReadBuffer(), addr: addr{ proto: proto, address: address, }, connID: connID, session: session, - buf: make(chan []byte, 1024), } return c } @@ -55,9 +50,6 @@ func (c *connection) tunnelClose(err error) { } func (c *connection) doTunnelClose(err error) { - c.Lock() - defer c.Unlock() - if c.err != nil { return } @@ -67,11 +59,11 @@ func (c *connection) doTunnelClose(err error) { c.err = io.ErrClosedPipe } - close(c.buf) + c.buffer.Close(c.err) } -func (c *connection) tunnelWriter() io.Writer { - return chanWriter{conn: c, C: c.buf} +func (c *connection) OnData(m *message) error { + return c.buffer.Offer(m.body) } func (c *connection) Close() error { @@ -79,56 +71,23 @@ func (c *connection) Close() error { return nil } -func (c *connection) copyData(b []byte) int { - n := copy(b, c.readBuf) - c.readBuf = c.readBuf[n:] - return n -} - func (c *connection) Read(b []byte) (int, error) { - if len(b) == 0 { - return 0, nil - } - - n := c.copyData(b) - if n > 0 { - return n, nil - } - - next, ok := <-c.buf - if !ok { - err := io.EOF - c.Lock() - if c.err != nil { - err = c.err - } - c.Unlock() - return 0, err - } - - c.readBuf = next - n = c.copyData(b) - return n, nil + n, err := c.buffer.Read(b) + return n, err } func (c *connection) Write(b []byte) (int, error) { - c.Lock() if c.err != nil { - defer c.Unlock() - return 0, c.err + return 0, io.ErrClosedPipe } - c.Unlock() - - deadline := int64(0) - if !c.writeDeadline.IsZero() { - deadline = time.Until(c.writeDeadline).Nanoseconds() / 1000000 - } - return c.session.writeMessage(newMessage(c.connID, deadline, b)) + msg := newMessage(c.connID, b) + return c.session.writeMessage(c.writeDeadline, msg) } func (c *connection) writeErr(err error) { if err != nil { - c.session.writeMessage(newErrorMessage(c.connID, err)) + msg := newErrorMessage(c.connID, err) + c.session.writeMessage(c.writeDeadline, msg) } } @@ -148,6 +107,7 @@ func (c *connection) SetDeadline(t time.Time) error { } func (c *connection) SetReadDeadline(t time.Time) error { + c.buffer.deadline = t return nil } @@ -168,34 +128,3 @@ func (a addr) Network() string { func (a addr) String() string { return a.address } - -type chanWriter struct { - conn *connection - C chan []byte -} - -func (c chanWriter) Write(buf []byte) (int, error) { - c.conn.Lock() - defer c.conn.Unlock() - - if c.conn.err != nil { - return 0, c.conn.err - } - - newBuf := make([]byte, len(buf)) - copy(newBuf, buf) - buf = newBuf - - select { - // must copy the buffer - case c.C <- buf: - return len(buf), nil - default: - select { - case c.C <- buf: - return len(buf), nil - case <-time.After(15 * time.Second): - return 0, errors.New("backed up reader") - } - } -} diff --git a/pkg/tool/remotedialer/dialer.go b/pkg/tool/remotedialer/dialer.go index 5772fa0814be1bf6a2222a07bb13b3c5ef3e6c85..3e669083afa9d9209791bc58251eb1dee32ac3b8 100644 --- a/pkg/tool/remotedialer/dialer.go +++ b/pkg/tool/remotedialer/dialer.go @@ -17,28 +17,24 @@ limitations under the License. package remotedialer import ( + "context" "net" - "time" ) -type Dialer func(network, address string) (net.Conn, error) +type Dialer func(ctx context.Context, network, address string) (net.Conn, error) func (s *Server) HasSession(clientKey string) bool { - _, err := s.sessions.getDialer(clientKey, 0) + _, err := s.sessions.getDialer(clientKey) return err == nil } -func (s *Server) Dial(clientKey string, deadline time.Duration, proto, address string) (net.Conn, error) { - d, err := s.sessions.getDialer(clientKey, deadline) - if err != nil { - return nil, err - } - - return d(proto, address) -} +func (s *Server) Dialer(clientKey string) Dialer { + return func(ctx context.Context, network, address string) (net.Conn, error) { + d, err := s.sessions.getDialer(clientKey) + if err != nil { + return nil, err + } -func (s *Server) Dialer(clientKey string, deadline time.Duration) Dialer { - return func(proto, address string) (net.Conn, error) { - return s.Dial(clientKey, deadline, proto, address) + return d(ctx, network, address) } } diff --git a/pkg/tool/remotedialer/message.go b/pkg/tool/remotedialer/message.go index 90e21e554662d3625691282dd69e4886191a8842..04c073da7d2b84e64e514dc25ba984416985c7b3 100644 --- a/pkg/tool/remotedialer/message.go +++ b/pkg/tool/remotedialer/message.go @@ -40,7 +40,8 @@ const ( ) var ( - idCounter int64 + idCounter int64 + legacyDeadline = (15 * time.Second).Milliseconds() ) func init() { @@ -54,7 +55,6 @@ type message struct { id int64 err error connID int64 - deadline int64 messageType messageType bytes []byte body io.Reader @@ -66,21 +66,19 @@ func nextid() int64 { return atomic.AddInt64(&idCounter, 1) } -func newMessage(connID int64, deadline int64, bytes []byte) *message { +func newMessage(connID int64, bytes []byte) *message { return &message{ id: nextid(), connID: connID, - deadline: deadline, messageType: Data, bytes: bytes, } } -func newConnect(connID int64, deadline time.Duration, proto, address string) *message { +func newConnect(connID int64, proto, address string) *message { return &message{ id: nextid(), connID: connID, - deadline: deadline.Nanoseconds() / 1000000, messageType: Connect, bytes: []byte(fmt.Sprintf("%s/%s", proto, address)), proto: proto, @@ -142,11 +140,11 @@ func newServerMessage(reader io.Reader) (*message, error) { } if m.messageType == Data || m.messageType == Connect { - deadline, err := binary.ReadVarint(buf) + // no longer used, this is the deadline field + _, err := binary.ReadVarint(buf) if err != nil { return nil, err } - m.deadline = deadline } if m.messageType == Connect { @@ -192,17 +190,17 @@ func (m *message) Err() error { } func (m *message) Bytes() []byte { - return append(m.header(), m.bytes...) + return append(m.header(len(m.bytes)), m.bytes...) } -func (m *message) header() []byte { - buf := make([]byte, 24) +func (m *message) header(space int) []byte { + buf := make([]byte, 24+space) offset := 0 offset += binary.PutVarint(buf[offset:], m.id) offset += binary.PutVarint(buf[offset:], m.connID) offset += binary.PutVarint(buf[offset:], int64(m.messageType)) if m.messageType == Data || m.messageType == Connect { - offset += binary.PutVarint(buf[offset:], m.deadline) + offset += binary.PutVarint(buf[offset:], legacyDeadline) } return buf[:offset] } @@ -211,8 +209,8 @@ func (m *message) Read(p []byte) (int, error) { return m.body.Read(p) } -func (m *message) WriteTo(wsConn *wsConn) (int, error) { - err := wsConn.WriteMessage(websocket.BinaryMessage, m.Bytes()) +func (m *message) WriteTo(deadline time.Time, wsConn *wsConn) (int, error) { + err := wsConn.WriteMessage(websocket.BinaryMessage, deadline, m.Bytes()) return len(m.bytes), err } @@ -226,7 +224,7 @@ func (m *message) String() string { case Error: return fmt.Sprintf("%d ERROR [%d]: %s", m.id, m.connID, m.Err()) case Connect: - return fmt.Sprintf("%d CONNECT [%d]: %s/%s deadline %d", m.id, m.connID, m.proto, m.address, m.deadline) + return fmt.Sprintf("%d CONNECT [%d]: %s/%s", m.id, m.connID, m.proto, m.address) case AddClient: return fmt.Sprintf("%d ADDCLIENT [%s]", m.id, m.address) case RemoveClient: diff --git a/pkg/tool/remotedialer/peer.go b/pkg/tool/remotedialer/peer.go index ba6e9eff7ae4cebc8965ee6aa3314a5585c31b46..f280bf2986eeb74ad0518dcd82a274f21058be96 100644 --- a/pkg/tool/remotedialer/peer.go +++ b/pkg/tool/remotedialer/peer.go @@ -95,7 +95,7 @@ func (p *peer) start(ctx context.Context, s *Server) { TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, }, - HandshakeTimeout: 10 * time.Second, + HandshakeTimeout: HandshakeTimeOut, } outer: @@ -114,16 +114,17 @@ outer: } session := NewClientSession(func(string, string) bool { return true }, ws) - session.dialer = func(network, address string) (net.Conn, error) { + session.dialer = func(ctx context.Context, network, address string) (net.Conn, error) { parts := strings.SplitN(network, "::", 2) if len(parts) != 2 { return nil, fmt.Errorf("invalid clientKey/proto: %s", network) } - return s.Dial(parts[0], 15*time.Second, parts[1], address) + d := s.Dialer(parts[0]) + return d(ctx, parts[1], address) } s.sessions.addListener(session) - _, err = session.Serve() + _, err = session.Serve(ctx) s.sessions.removeListener(session) session.Close() diff --git a/pkg/tool/remotedialer/readbuffer.go b/pkg/tool/remotedialer/readbuffer.go new file mode 100644 index 0000000000000000000000000000000000000000..3b2f5498c8e91d93fa6c253120742e02254f1e4f --- /dev/null +++ b/pkg/tool/remotedialer/readbuffer.go @@ -0,0 +1,122 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package remotedialer + +import ( + "bytes" + "errors" + "io" + "sync" + "time" +) + +const ( + MaxBuffer = 1 << 20 +) + +type readBuffer struct { + cond sync.Cond + deadline time.Time + buf bytes.Buffer + err error +} + +func newReadBuffer() *readBuffer { + return &readBuffer{ + cond: sync.Cond{ + L: &sync.Mutex{}, + }, + } +} + +func (r *readBuffer) Offer(reader io.Reader) error { + r.cond.L.Lock() + defer r.cond.L.Unlock() + + for { + if r.err != nil { + return r.err + } + + if n, err := io.Copy(&r.buf, reader); err != nil { + return err + } else if n > 0 { + r.cond.Broadcast() + } + + if r.buf.Len() < MaxBuffer { + return nil + } + + r.cond.Wait() + } +} + +func (r *readBuffer) Read(b []byte) (int, error) { + r.cond.L.Lock() + defer r.cond.L.Unlock() + + for { + var ( + n int + err error + ) + + if r.buf.Len() > 0 { + n, err = r.buf.Read(b) + r.cond.Broadcast() + if err != io.EOF { + return n, err + } + // buffer remains to be read + if r.err == io.EOF && r.buf.Len() > 0 { + return n, nil + } + return n, r.err + } + + if r.err != nil { + return n, r.err + } + + now := time.Now() + if !r.deadline.IsZero() { + if now.After(r.deadline) { + return 0, errors.New("deadline exceeded") + } + } + + var t *time.Timer + if !r.deadline.IsZero() { + t = time.AfterFunc(r.deadline.Sub(now), func() { r.cond.Broadcast() }) + } + r.cond.Wait() + if t != nil { + t.Stop() + } + } +} + +func (r *readBuffer) Close(err error) error { + r.cond.L.Lock() + defer r.cond.L.Unlock() + if r.err == nil { + r.err = err + } + r.cond.Broadcast() + return nil +} diff --git a/pkg/tool/remotedialer/server.go b/pkg/tool/remotedialer/server.go index d54015675f2749ff642f7bf71aebc7239d10c348..d9806abdaa19ef0875b65457c91f3aaf839a206e 100644 --- a/pkg/tool/remotedialer/server.go +++ b/pkg/tool/remotedialer/server.go @@ -17,6 +17,9 @@ limitations under the License. package remotedialer import ( + "crypto/tls" + "crypto/x509" + "encoding/base64" "net/http" "sync" "time" @@ -36,17 +39,26 @@ type ErrorWriter func(rw http.ResponseWriter, req *http.Request, code int, err e func DefaultErrorWriter(rw http.ResponseWriter, req *http.Request, code int, err error) { rw.WriteHeader(code) - rw.Write([]byte(err.Error())) + _, errWrite := rw.Write([]byte(err.Error())) + if errWrite != nil { + logrus.Warnf("Failed to write error: %s", errWrite) + } } type Server struct { - PeerID string - PeerToken string - authorizer Authorizer - errorWriter ErrorWriter - sessions *sessionManager - peers map[string]peer - peerLock sync.Mutex + PeerID string + PeerToken string + ClientConnectAuthorizer ConnectAuthorizer + authorizer Authorizer + errorWriter ErrorWriter + sessions *sessionManager + peers map[string]peer + peerLock sync.Mutex + + caCert string + httpTransport *http.Transport + + sync.Mutex } func New(auth Authorizer, errorWriter ErrorWriter) *Server { @@ -84,10 +96,10 @@ func (s *Server) ServeHTTP(rw http.ResponseWriter, req *http.Request) { } session := s.sessions.add(clientKey, wsConn, peer) + session.auth = s.ClientConnectAuthorizer defer s.sessions.remove(session) - // Don't need to associate req.Context() to the Session, it will cancel otherwise - code, err := session.Serve() + code, err := session.Serve(req.Context()) if err != nil { // Hijacked so we can't write to the client logrus.Infof("error in remotedialer server [%d]: %v", code, err) @@ -112,6 +124,38 @@ func (s *Server) auth(req *http.Request) (clientKey string, authed, peer bool, e return id, authed, false, err } +func (r *Server) GetTransport(clusterCaCert string, clientKey string) (http.RoundTripper, error) { + + r.Lock() + defer r.Unlock() + + transport := &http.Transport{} + if clusterCaCert != "" { + certBytes, err := base64.StdEncoding.DecodeString(clusterCaCert) + if err != nil { + return nil, err + } + certs := x509.NewCertPool() + certs.AppendCertsFromPEM(certBytes) + transport.TLSClientConfig = &tls.Config{ + RootCAs: certs, + } + } + + d := r.Dialer(clientKey) + transport.DialContext = d + transport.Proxy = http.ProxyFromEnvironment + transport.IdleConnTimeout = 30 * time.Second + + r.caCert = clusterCaCert + if r.httpTransport != nil { + r.httpTransport.CloseIdleConnections() + } + r.httpTransport = transport + + return transport, nil +} + func (s *Server) Disconnect(clientKey string) { for _, session := range s.sessions.clients[clientKey] { session.CloseImmediately() diff --git a/pkg/tool/remotedialer/session.go b/pkg/tool/remotedialer/session.go index 82d017a545ca8c800a2332b637e8716e9840258e..0baaec170205aad2780ba00e6dd429076a8f49b2 100644 --- a/pkg/tool/remotedialer/session.go +++ b/pkg/tool/remotedialer/session.go @@ -59,12 +59,17 @@ func init() { } func NewClientSession(auth ConnectAuthorizer, conn *websocket.Conn) *Session { + return NewClientSessionWithDialer(auth, conn, nil) +} + +func NewClientSessionWithDialer(auth ConnectAuthorizer, conn *websocket.Conn, dialer Dialer) *Session { return &Session{ clientKey: "client", conn: newWSConn(conn), conns: map[int64]*connection{}, auth: auth, client: true, + dialer: dialer, } } @@ -79,8 +84,8 @@ func newSession(sessionKey int64, clientKey string, conn *websocket.Conn) *Sessi } } -func (s *Session) startPings() { - ctx, cancel := context.WithCancel(context.Background()) +func (s *Session) startPings(rootCtx context.Context) { + ctx, cancel := context.WithCancel(rootCtx) s.pingCancel = cancel s.pingWait.Add(1) @@ -96,7 +101,7 @@ func (s *Session) startPings() { return case <-t.C: s.conn.Lock() - if err := s.conn.conn.WriteControl(websocket.PingMessage, []byte(""), time.Now().Add(time.Second)); err != nil { + if err := s.conn.conn.WriteControl(websocket.PingMessage, []byte(""), time.Now().Add(PingWaitDuration)); err != nil { logrus.WithError(err).Error("Error writing ping") } logrus.Debug("Wrote ping") @@ -115,9 +120,9 @@ func (s *Session) stopPings() { s.pingWait.Wait() } -func (s *Session) Serve() (int, error) { +func (s *Session) Serve(ctx context.Context) (int, error) { if s.client { - s.startPings() + s.startPings(ctx) } for { @@ -130,13 +135,13 @@ func (s *Session) Serve() (int, error) { return 400, errWrongMessageType } - if err := s.serveMessage(reader); err != nil { + if err := s.serveMessage(ctx, reader); err != nil { return 500, err } } } -func (s *Session) serveMessage(reader io.Reader) error { +func (s *Session) serveMessage(ctx context.Context, reader io.Reader) error { message, err := newServerMessage(reader) if err != nil { return err @@ -150,7 +155,7 @@ func (s *Session) serveMessage(reader io.Reader) error { if s.auth == nil || !s.auth(message.proto, message.address) { return errors.New("connect not allowed") } - s.clientConnect(message) + s.clientConnect(ctx, message) return nil } @@ -170,14 +175,14 @@ func (s *Session) serveMessage(reader io.Reader) error { if conn == nil { if message.messageType == Data { err := fmt.Errorf("connection not found %s/%d/%d", s.clientKey, s.sessionKey, message.connID) - newErrorMessage(message.connID, err).WriteTo(s.conn) + newErrorMessage(message.connID, err).WriteTo(defaultDeadline(), s.conn) } return nil } switch message.messageType { case Data: - if _, err := io.Copy(conn.tunnelWriter(), message); err != nil { + if err := conn.OnData(message); err != nil { s.closeConnection(message.connID, err) } case Error: @@ -187,6 +192,10 @@ func (s *Session) serveMessage(reader io.Reader) error { return nil } +func defaultDeadline() time.Time { + return time.Now().Add(time.Minute) +} + func parseAddress(address string) (string, int, error) { parts := strings.SplitN(address, "/", 2) if len(parts) != 2 { @@ -207,7 +216,7 @@ func (s *Session) addRemoteClient(address string) error { keys = map[int]bool{} s.remoteClientKeys[clientKey] = keys } - keys[int(sessionKey)] = true + keys[sessionKey] = true if PrintTunnelData { logrus.Debugf("ADD REMOTE CLIENT %s, SESSION %d", address, s.sessionKey) @@ -249,7 +258,7 @@ func (s *Session) closeConnection(connID int64, err error) { } } -func (s *Session) clientConnect(message *message) { +func (s *Session) clientConnect(ctx context.Context, message *message) { conn := newConnection(message.connID, s, message.proto, message.address) s.Lock() @@ -259,10 +268,46 @@ func (s *Session) clientConnect(message *message) { } s.Unlock() - go clientDial(s.dialer, conn, message) + go clientDial(ctx, s.dialer, conn, message) +} + +type connResult struct { + conn net.Conn + err error +} + +func (s *Session) Dial(ctx context.Context, proto, address string) (net.Conn, error) { + return s.serverConnectContext(ctx, proto, address) } -func (s *Session) serverConnect(deadline time.Duration, proto, address string) (net.Conn, error) { +func (s *Session) serverConnectContext(ctx context.Context, proto, address string) (net.Conn, error) { + deadline, ok := ctx.Deadline() + if ok { + return s.serverConnect(deadline, proto, address) + } + + result := make(chan connResult, 1) + go func() { + c, err := s.serverConnect(defaultDeadline(), proto, address) + result <- connResult{conn: c, err: err} + }() + + select { + case <-ctx.Done(): + // We don't want to orphan an open connection so we wait for the result and immediately close it + go func() { + r := <-result + if r.err == nil { + r.conn.Close() + } + }() + return nil, ctx.Err() + case r := <-result: + return r.conn, r.err + } +} + +func (s *Session) serverConnect(deadline time.Time, proto, address string) (net.Conn, error) { connID := atomic.AddInt64(&s.nextConnID, 1) conn := newConnection(connID, s, proto, address) @@ -273,7 +318,7 @@ func (s *Session) serverConnect(deadline time.Duration, proto, address string) ( } s.Unlock() - _, err := s.writeMessage(newConnect(connID, deadline, proto, address)) + _, err := s.writeMessage(deadline, newConnect(connID, proto, address)) if err != nil { s.closeConnection(connID, err) return nil, err @@ -282,11 +327,11 @@ func (s *Session) serverConnect(deadline time.Duration, proto, address string) ( return conn, err } -func (s *Session) writeMessage(message *message) (int, error) { +func (s *Session) writeMessage(deadline time.Time, message *message) (int, error) { if PrintTunnelData { logrus.Debug("WRITE ", message) } - return message.WriteTo(s.conn) + return message.WriteTo(deadline, s.conn) } func (s *Session) Close() { @@ -302,14 +347,9 @@ func (s *Session) Close() { s.conns = map[int64]*connection{} } -func (s *Session) CloseImmediately() { - s.Close() - _ = s.conn.conn.Close() -} - func (s *Session) sessionAdded(clientKey string, sessionKey int64) { client := fmt.Sprintf("%s/%d", clientKey, sessionKey) - _, err := s.writeMessage(newAddClient(client)) + _, err := s.writeMessage(time.Time{}, newAddClient(client)) if err != nil { s.conn.conn.Close() } @@ -317,8 +357,13 @@ func (s *Session) sessionAdded(clientKey string, sessionKey int64) { func (s *Session) sessionRemoved(clientKey string, sessionKey int64) { client := fmt.Sprintf("%s/%d", clientKey, sessionKey) - _, err := s.writeMessage(newRemoveClient(client)) + _, err := s.writeMessage(time.Time{}, newRemoveClient(client)) if err != nil { s.conn.conn.Close() } } + +func (s *Session) CloseImmediately() { + s.Close() + _ = s.conn.conn.Close() +} diff --git a/pkg/tool/remotedialer/session_manager.go b/pkg/tool/remotedialer/session_manager.go index d92062ab93362ed7c1f93b7bc9d4f19406885b35..2fab0bd737c08b2f4a456e1ec23765cc9f63413a 100644 --- a/pkg/tool/remotedialer/session_manager.go +++ b/pkg/tool/remotedialer/session_manager.go @@ -17,11 +17,11 @@ limitations under the License. package remotedialer import ( + "context" "fmt" "math/rand" "net" "sync" - "time" "github.com/gorilla/websocket" ) @@ -46,12 +46,12 @@ func newSessionManager() *sessionManager { } } -func toDialer(s *Session, prefix string, deadline time.Duration) Dialer { - return func(proto, address string) (net.Conn, error) { +func toDialer(s *Session, prefix string) Dialer { + return func(ctx context.Context, proto, address string) (net.Conn, error) { if prefix == "" { - return s.serverConnect(deadline, proto, address) + return s.serverConnectContext(ctx, proto, address) } - return s.serverConnect(deadline, prefix+"::"+proto, address) + return s.serverConnectContext(ctx, prefix+"::"+proto, address) } } @@ -81,13 +81,13 @@ func (sm *sessionManager) addListener(listener sessionListener) { } } -func (sm *sessionManager) getDialer(clientKey string, deadline time.Duration) (Dialer, error) { +func (sm *sessionManager) getDialer(clientKey string) (Dialer, error) { sm.Lock() defer sm.Unlock() sessions := sm.clients[clientKey] if len(sessions) > 0 { - return toDialer(sessions[0], "", deadline), nil + return toDialer(sessions[0], ""), nil } for _, sessions := range sm.peers { @@ -96,7 +96,7 @@ func (sm *sessionManager) getDialer(clientKey string, deadline time.Duration) (D keys := session.remoteClientKeys[clientKey] session.Unlock() if len(keys) > 0 { - return toDialer(session, clientKey, deadline), nil + return toDialer(session, clientKey), nil } } } diff --git a/pkg/tool/remotedialer/types.go b/pkg/tool/remotedialer/types.go index 061408d3ff11e6e6c6fb28069aa9caff1095c4b3..98ed9dfd040e66db37100ec4bcb6cacf7550fc4b 100644 --- a/pkg/tool/remotedialer/types.go +++ b/pkg/tool/remotedialer/types.go @@ -16,12 +16,11 @@ limitations under the License. package remotedialer -import ( - "time" -) +import "time" -var ( - PingWaitDuration = time.Duration(10 * time.Second) - PingWriteInterval = time.Duration(5 * time.Second) +const ( + PingWaitDuration = 60 * time.Second + PingWriteInterval = 5 * time.Second MaxRead = 8192 + HandshakeTimeOut = 10 * time.Second ) diff --git a/pkg/tool/remotedialer/wsconn.go b/pkg/tool/remotedialer/wsconn.go index 51256698c4c00f9ae7cf795e6f65e6075ad1c716..9a53ddc8852d58975e9024e92e65a19c032ce617 100644 --- a/pkg/tool/remotedialer/wsconn.go +++ b/pkg/tool/remotedialer/wsconn.go @@ -17,6 +17,8 @@ limitations under the License. package remotedialer import ( + "context" + "fmt" "io" "sync" "time" @@ -37,11 +39,29 @@ func newWSConn(conn *websocket.Conn) *wsConn { return w } -func (w *wsConn) WriteMessage(messageType int, data []byte) error { - w.Lock() - defer w.Unlock() - w.conn.SetWriteDeadline(time.Now().Add(PingWaitDuration)) - return w.conn.WriteMessage(messageType, data) +func (w *wsConn) WriteMessage(messageType int, deadline time.Time, data []byte) error { + if deadline.IsZero() { + w.Lock() + defer w.Unlock() + return w.conn.WriteMessage(messageType, data) + } + + ctx, cancel := context.WithDeadline(context.Background(), deadline) + defer cancel() + + done := make(chan error, 1) + go func() { + w.Lock() + defer w.Unlock() + done <- w.conn.WriteMessage(messageType, data) + }() + + select { + case <-ctx.Done(): + return fmt.Errorf("i/o timeout") + case err := <-done: + return err + } } func (w *wsConn) NextReader() (int, io.Reader, error) { @@ -52,12 +72,21 @@ func (w *wsConn) setupDeadline() { w.conn.SetReadDeadline(time.Now().Add(PingWaitDuration)) w.conn.SetPingHandler(func(string) error { w.Lock() - w.conn.WriteControl(websocket.PongMessage, []byte(""), time.Now().Add(time.Second)) + err := w.conn.WriteControl(websocket.PongMessage, []byte(""), time.Now().Add(PingWaitDuration)) w.Unlock() - return w.conn.SetReadDeadline(time.Now().Add(PingWaitDuration)) + if err != nil { + return err + } + if err := w.conn.SetReadDeadline(time.Now().Add(PingWaitDuration)); err != nil { + return err + } + return w.conn.SetWriteDeadline(time.Now().Add(PingWaitDuration)) }) w.conn.SetPongHandler(func(string) error { - return w.conn.SetReadDeadline(time.Now().Add(PingWaitDuration)) + if err := w.conn.SetReadDeadline(time.Now().Add(PingWaitDuration)); err != nil { + return err + } + return w.conn.SetWriteDeadline(time.Now().Add(PingWaitDuration)) }) } diff --git a/pkg/tool/rsa/rsa.go b/pkg/tool/rsa/rsa.go index fcae3b451d6ba868ce2aa0f1c24d96f1546e2e68..1128341e2dab4b379b022a6f2d3a3b8acc1b2c61 100644 --- a/pkg/tool/rsa/rsa.go +++ b/pkg/tool/rsa/rsa.go @@ -23,6 +23,8 @@ import ( "encoding/pem" "errors" "io/ioutil" + "log" + "runtime" ) //var defaultPrivateKey = []byte(` @@ -154,6 +156,87 @@ func LoadPubKey(filename string) (err error) { return } +const ( + privateKeyPrefix = "RSA PRIVATE KEY" + publicKeyPrefix = "RSA PUBLIC KEY" +) + +func EncryptByPublicKey(plainText, key []byte) (cryptText []byte, err error) { + block, _ := pem.Decode(key) + defer func() { + if err := recover(); err != nil { + switch err.(type) { + case runtime.Error: + log.Println("runtime err:", err, "Check that the key is correct") + default: + log.Println("error:", err) + } + } + }() + publicKeyInterface, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + return nil, err + } + publicKey := publicKeyInterface.(*rsa.PublicKey) + + cipherText, err := rsa.EncryptPKCS1v15(rand.Reader, publicKey, plainText) + if err != nil { + return nil, err + } + return cipherText, nil +} + +func DecryptByPrivateKey(cryptText, key []byte) (plainText []byte, err error) { + block, _ := pem.Decode(key) + + defer func() { + if err := recover(); err != nil { + switch err.(type) { + case runtime.Error: + log.Println("runtime err:", err, "Check that the key is correct") + default: + log.Println("error:", err) + } + } + }() + privateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + return []byte{}, err + } + plainText, err = rsa.DecryptPKCS1v15(rand.Reader, privateKey, cryptText) + if err != nil { + return []byte{}, err + } + return plainText, nil +} + +func GetRsaKey() (error, []byte, []byte) { + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return err, nil, nil + } + x509PrivateKey := x509.MarshalPKCS1PrivateKey(privateKey) + privateBlock := pem.Block{ + Type: privateKeyPrefix, + Bytes: x509PrivateKey, + } + privateKeyRes := pem.EncodeToMemory(&privateBlock) + + publicKey := privateKey.PublicKey + x509PublicKey, err := x509.MarshalPKIXPublicKey(&publicKey) + if err != nil { + return err, nil, nil + } + + publicBlock := pem.Block{ + Type: publicKeyPrefix, + Bytes: x509PublicKey, + } + + publicKeyRes := pem.EncodeToMemory(&publicBlock) + return nil, publicKeyRes, privateKeyRes +} + //RSA解密 // cipherText 需要解密的byte数据 func Decrypt(cipherText []byte) ([]byte, error) { diff --git a/pkg/tool/s3/client.go b/pkg/tool/s3/client.go index de4076b6d0f7871aa59b96e06606a12e14e1aa0f..44b8da8ee127c4618e0a7c0dfcea450d564070f9 100644 --- a/pkg/tool/s3/client.go +++ b/pkg/tool/s3/client.go @@ -18,8 +18,10 @@ package s3 import ( "fmt" + "io/fs" + "mime" "os" - "path" + "path/filepath" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" @@ -28,7 +30,7 @@ import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/koderover/zadig/pkg/tool/log" - "github.com/koderover/zadig/pkg/util/fs" + fsutil "github.com/koderover/zadig/pkg/util/fs" ) const ( @@ -39,6 +41,15 @@ type Client struct { *s3.S3 } +type DownloadOption struct { + IgnoreNotExistError bool + RetryNum int +} + +var defaultDownloadOption = &DownloadOption{ + RetryNum: 3, +} + func NewClient(endpoint, ak, sk string, insecure, forcedPathStyle bool) (*Client, error) { creds := credentials.NewStaticCredentials(ak, sk, "") config := &aws.Config{ @@ -57,26 +68,30 @@ func NewClient(endpoint, ak, sk string, insecure, forcedPathStyle bool) (*Client // Validate the existence of bucket func (c *Client) ValidateBucket(bucketName string) error { - listBucketInput := &s3.ListBucketsInput{} - bucketListResp, err := c.ListBuckets(listBucketInput) + listObjectInput := &s3.ListObjectsInput{Bucket: aws.String(bucketName)} + _, err := c.ListObjects(listObjectInput) if err != nil { return fmt.Errorf("validate S3 error: %s", err.Error()) } - for _, bucket := range bucketListResp.Buckets { - if *bucket.Name == bucketName { - return nil - } - } - return fmt.Errorf("validate s3 error: given bucket does not exist") + return nil +} + +func (c *Client) DownloadWithOption(bucketName, objectKey, dest string, option *DownloadOption) error { + return c.download(bucketName, objectKey, dest, option) } // Download the file to object storage func (c *Client) Download(bucketName, objectKey, dest string) error { + return c.download(bucketName, objectKey, dest, defaultDownloadOption) +} + +func (c *Client) download(bucketName, objectKey, dest string, option *DownloadOption) error { + retry := 0 var err error - for retry < 3 { + for retry < option.RetryNum { opt := &s3.GetObjectInput{ Bucket: aws.String(bucketName), Key: aws.String(objectKey), @@ -84,6 +99,9 @@ func (c *Client) Download(bucketName, objectKey, dest string) error { obj, err1 := c.GetObject(opt) if err1 != nil { if e, ok := err1.(awserr.Error); ok && e.Code() == s3.ErrCodeNoSuchKey { + if option.IgnoreNotExistError { + return nil + } return err1 } @@ -93,7 +111,7 @@ func (c *Client) Download(bucketName, objectKey, dest string) error { retry++ continue } - err = fs.SaveFile(obj.Body, dest) + err = fsutil.SaveFile(obj.Body, dest) if err != nil { log.Errorf("Failed to save file to %s, err: %s", dest, err) } @@ -176,6 +194,14 @@ func (c *Client) RemoveFiles(bucketName string, prefixList []string) { } } +func detectMimetype(path string) string { + fileext := filepath.Ext(path) + if fileext == "" { + return "" + } + return mime.TypeByExtension(fileext) +} + // Upload uploads a file from src to the bucket with the specified objectKey func (c *Client) Upload(bucketName, src string, objectKey string) error { file, err := os.OpenFile(src, os.O_RDONLY, 0600) @@ -188,10 +214,59 @@ func (c *Client) Upload(bucketName, src string, objectKey string) error { Bucket: aws.String(bucketName), Key: aws.String(objectKey), } + mimetype := detectMimetype(src) + if mimetype != "" { + input.ContentType = &mimetype + } _, err = c.PutObject(input) return err } +// Upload upload all files in a directory to a S3 path recursively +func (c *Client) UploadDir(bucketName, srcdir string, s3dir string) error { + err := fs.WalkDir(os.DirFS(srcdir), ".", func(p string, d fs.DirEntry, e error) error { + if e != nil { + return e + } + if d.IsDir() { + return nil + } + key := filepath.Join(s3dir, p) + originalFilePath := filepath.Join(srcdir, p) + return c.Upload(bucketName, originalFilePath, key) + }) + return err +} + +func (c *Client) GetFile(bucketName, objectKey string, option *DownloadOption) (*s3.GetObjectOutput, error) { + retry := 0 + var err error + + for retry < option.RetryNum { + opt := &s3.GetObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + } + obj, err1 := c.GetObject(opt) + if err1 != nil { + if e, ok := err1.(awserr.Error); ok && e.Code() == s3.ErrCodeNoSuchKey { + if option.IgnoreNotExistError { + return nil, nil + } + return nil, err1 + } + + log.Warnf("Failed to get object %s from s3, try again, err: %s", objectKey, err1) + err = err1 + retry++ + continue + } + + return obj, nil + } + return nil, err +} + // ListFiles with given prefix func (c *Client) ListFiles(bucketName, prefix string, recursive bool) ([]string, error) { ret := make([]string, 0) @@ -211,8 +286,7 @@ func (c *Client) ListFiles(bucketName, prefix string, recursive bool) ([]string, for _, item := range output.Contents { itemKey := *item.Key - _, fileName := path.Split(itemKey) - ret = append(ret, fileName) + ret = append(ret, itemKey) } return ret, nil diff --git a/pkg/tool/s3/client_test.go b/pkg/tool/s3/client_test.go new file mode 100644 index 0000000000000000000000000000000000000000..9c40a6c2aaeacf8aff7161f4c1da892e3effb940 --- /dev/null +++ b/pkg/tool/s3/client_test.go @@ -0,0 +1,43 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package s3 + +import ( + "testing" +) + +func TestDetectMimetype(t *testing.T) { + type testcase struct { + file string + mimetype string + } + testcases := []testcase{ + {"", ""}, + {"/a/abc.", ""}, + {"abc/ab", ""}, + {".abc.a", ""}, + {"./abc.js", "application/javascript"}, + {"./abc.css", "text/css; charset=utf-8"}, + {"./abc.html", "text/html; charset=utf-8"}, + } + for _, ts := range testcases { + m := detectMimetype(ts.file) + if m != ts.mimetype { + t.Errorf("Expected result for path <%s> to be <%s> but got <%s>", ts.file, ts.mimetype, m) + } + } +} diff --git a/pkg/tool/ssh/ssh.go b/pkg/tool/ssh/ssh.go new file mode 100644 index 0000000000000000000000000000000000000000..da38b4b1288a1ed2f27f4f4ca22366843c6402e4 --- /dev/null +++ b/pkg/tool/ssh/ssh.go @@ -0,0 +1,51 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ssh + +import ( + "fmt" + "net" + "time" + + "golang.org/x/crypto/ssh" +) + +func NewSshCli(privateKey []byte, user, host string, port int64) (*ssh.Client, error) { + var ( + clientConfig *ssh.ClientConfig + client *ssh.Client + err error + ) + + sign, err := ssh.ParsePrivateKey(privateKey) + if err != nil { + return nil, err + } + auth := []ssh.AuthMethod{ssh.PublicKeys(sign)} + clientConfig = &ssh.ClientConfig{ + User: user, + Auth: auth, + Timeout: 5 * time.Second, + HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error { + return nil + }, + } + + addr := fmt.Sprintf("%s:%d", host, port) + client, err = ssh.Dial("tcp", addr, clientConfig) + return client, err +} diff --git a/pkg/tool/wsconn/wsconn.go b/pkg/tool/wsconn/wsconn.go new file mode 100644 index 0000000000000000000000000000000000000000..c80cd40c02378286943ead6730145fb0f6613c06 --- /dev/null +++ b/pkg/tool/wsconn/wsconn.go @@ -0,0 +1,188 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wsconn + +import ( + "bytes" + "encoding/json" + "io" + "sync" + "time" + + "github.com/gorilla/websocket" + "golang.org/x/crypto/ssh" + + "github.com/koderover/zadig/pkg/tool/log" +) + +type wsOperationType string + +const ( + wsMsgStdout wsOperationType = "stdout" + wsMsgStdin wsOperationType = "stdin" + wsMsgResize wsOperationType = "resize" +) + +type wsMessage struct { + Operation wsOperationType `json:"operation"` + Data string `json:"data"` + Rows int `json:"rows"` + Cols int `json:"cols"` +} + +type wsBufferWriter struct { + buffer bytes.Buffer + mu sync.Mutex +} + +func (w *wsBufferWriter) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + return w.buffer.Write(p) +} + +type SshConn struct { + Stdin io.WriteCloser + WsWriter *wsBufferWriter + SshSession *ssh.Session +} + +func NewSshConn(cols, rows int, sshClient *ssh.Client) (*SshConn, error) { + sshSession, err := sshClient.NewSession() + if err != nil { + return nil, err + } + + stdin, err := sshSession.StdinPipe() + if err != nil { + return nil, err + } + + wsWriter := new(wsBufferWriter) + sshSession.Stdout = wsWriter + sshSession.Stderr = wsWriter + + modes := ssh.TerminalModes{ + ssh.ECHO: 1, + ssh.TTY_OP_ISPEED: 14400, + ssh.TTY_OP_OSPEED: 14400, + } + + if err := sshSession.RequestPty("xterm", rows, cols, modes); err != nil { + return nil, err + } + if err := sshSession.Shell(); err != nil { + return nil, err + } + return &SshConn{Stdin: stdin, WsWriter: wsWriter, SshSession: sshSession}, nil +} + +func (ssConn *SshConn) ReadWsMessage(wsConn *websocket.Conn, stopCh chan bool) { + defer setStop(stopCh) + for { + select { + case <-stopCh: + return + default: + _, wsData, err := wsConn.ReadMessage() + if err != nil { + log.Errorf("read ws message err:%s", err) + return + } + var wsMsgObj wsMessage + if err := json.Unmarshal(wsData, &wsMsgObj); err != nil { + log.Error("ReceiveWsMessage Unmarshal err:", string(wsData), err) + return + } + + switch wsMsgObj.Operation { + case wsMsgResize: + if wsMsgObj.Cols > 0 && wsMsgObj.Rows > 0 { + if err := ssConn.SshSession.WindowChange(wsMsgObj.Rows, wsMsgObj.Cols); err != nil { + log.Error("resize windows err:", err) + } + } + case wsMsgStdin: + decodeBytes := []byte(wsMsgObj.Data) + if _, err := ssConn.Stdin.Write(decodeBytes); err != nil { + log.Error("ws stdin write to ssh.stdin err:", err) + setStop(stopCh) + } + } + } + } +} + +func (ssConn *SshConn) SendWsWriteMessage(wsConn *websocket.Conn, stopCh chan bool) { + defer setStop(stopCh) + tick := time.NewTicker(time.Millisecond * time.Duration(12)) + pingTick := time.NewTimer(10 * time.Second) + defer tick.Stop() + + for { + select { + case <-tick.C: + if err := flushWsWriter(ssConn.WsWriter, wsConn); err != nil { + log.Error("ssh sending output to webSocket err:", err) + return + } + case <-pingTick.C: + if err := wsConn.WriteMessage(websocket.PingMessage, []byte{}); err != nil { + log.Error("ssh sending ping to webSocket err:", err) + return + } + case <-stopCh: + return + } + } +} + +func (ssConn *SshConn) SessionWait(stopChan chan bool) { + if err := ssConn.SshSession.Wait(); err != nil { + log.Error("ssh session wait err:", err) + setStop(stopChan) + } +} + +func (s *SshConn) Close() { + if s.SshSession != nil { + s.SshSession.Close() + } +} + +func setStop(ch chan bool) { + ch <- true +} + +func flushWsWriter(w *wsBufferWriter, wsConn *websocket.Conn) error { + if w.buffer.Len() != 0 { + msg, err := json.Marshal(wsMessage{ + Operation: wsMsgStdout, + Data: string(w.buffer.Bytes()), + }) + if err != nil { + log.Errorf("write parse message err: %s", err) + return err + } + err = wsConn.WriteMessage(websocket.TextMessage, msg) + if err != nil { + return err + } + w.buffer.Reset() + } + return nil +} diff --git a/pkg/types/build.go b/pkg/types/build.go new file mode 100644 index 0000000000000000000000000000000000000000..cdccd0e532a6da8b5283433c320ae101004e7474 --- /dev/null +++ b/pkg/types/build.go @@ -0,0 +1,32 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +type JenkinsBuildParam struct { + Name string `bson:"name,omitempty" json:"name,omitempty"` + Value interface{} `bson:"value,omitempty" json:"value,omitempty"` + Type JenkinsParamType `bson:"type,omitempty" json:"type,omitempty"` + AutoGenerate bool `bson:"auto_generate,omitempty" json:"auto_generate,omitempty"` + ChoiceOption []string `bson:"choice_option,omitempty" json:"choice_option,omitempty"` +} + +type JenkinsParamType string + +const ( + Str JenkinsParamType = "string" + Choice JenkinsParamType = "choice" +) diff --git a/pkg/types/cache.go b/pkg/types/cache.go new file mode 100644 index 0000000000000000000000000000000000000000..da0d8ad5667f5987a6856deca1fc5d1d4a908b7d --- /dev/null +++ b/pkg/types/cache.go @@ -0,0 +1,67 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +type MediumType string + +const ( + ObjectMedium MediumType = "object" + NFSMedium MediumType = "nfs" +) + +type ProvisionType string + +const ( + DynamicProvision ProvisionType = "dynamic" + StaticProvision ProvisionType = "static" +) + +// Note: +// In the current design, when the object storage is used as the cache medium, the default object storage +// will be used as the cache medium, but in the near future users may be allowed to select different object storage +// as the cache medium according to the cluster. +// So it is temporarily reserved in `v1.10.0`, but do not use. +type ObjectProperties struct { + ID string `json:"id" bson:"id"` +} + +type NFSProperties struct { + ProvisionType ProvisionType `json:"provision_type" bson:"provision_type"` + StorageClass string `json:"storage_class" bson:"storage_class"` + StorageSizeInGiB int64 `json:"storage_size_in_gib" bson:"storage_size_in_gib"` + PVC string `json:"pvc" bson:"pvc"` + Subpath string `json:"subpath" bson:"subpath"` +} + +type Cache struct { + MediumType MediumType `json:"medium_type" bson:"medium_type"` + ObjectProperties ObjectProperties `json:"object_properties" bson:"object_properties"` + NFSProperties NFSProperties `json:"nfs_properties" bson:"nfs_properties"` +} + +type CacheDirType string + +const ( + WorkspaceCacheDir CacheDirType = "workspace" + UserDefinedCacheDir CacheDirType = "user_defined" +) + +type StorageClassType string + +const ( + StorageClassAll StorageClassType = "all" +) diff --git a/pkg/types/cron.go b/pkg/types/cron.go new file mode 100644 index 0000000000000000000000000000000000000000..a1dd1bd5c37fa54e55fbae5debb807a4eb8f5cbf --- /dev/null +++ b/pkg/types/cron.go @@ -0,0 +1,23 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +type CronTag string + +const ( + CleanDockerTag CronTag = "CleanDockerTag" +) diff --git a/pkg/types/dto/workflowtask.go b/pkg/types/dto/workflowtask.go new file mode 100644 index 0000000000000000000000000000000000000000..21e998a6d6a710701418a500d80a05237dc1ebcb --- /dev/null +++ b/pkg/types/dto/workflowtask.go @@ -0,0 +1,91 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dto + +import ( + "sync" + + "go.mongodb.org/mongo-driver/bson/primitive" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/setting" +) + +type Task struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + TaskID int64 `bson:"task_id" json:"task_id"` + ProductName string `bson:"product_name" json:"product_name"` + PipelineName string `bson:"pipeline_name" json:"pipeline_name"` + Type config.PipelineType `bson:"type" json:"type"` + Status config.Status `bson:"status" json:"status,omitempty"` + Description string `bson:"description,omitempty" json:"description,omitempty"` + TaskCreator string `bson:"task_creator" json:"task_creator,omitempty"` + TaskRevoker string `bson:"task_revoker,omitempty" json:"task_revoker,omitempty"` + CreateTime int64 `bson:"create_time" json:"create_time,omitempty"` + StartTime int64 `bson:"start_time" json:"start_time,omitempty"` + EndTime int64 `bson:"end_time" json:"end_time,omitempty"` + SubTasks []map[string]interface{} `bson:"sub_tasks" json:"sub_tasks"` + Stages []*models.Stage `bson:"stages" json:"stages"` + ReqID string `bson:"req_id,omitempty" json:"req_id,omitempty"` + AgentHost string `bson:"agent_host,omitempty" json:"agent_host,omitempty"` + DockerHost string `bson:"-" json:"docker_host,omitempty"` + TeamName string `bson:"team,omitempty" json:"team,omitempty"` + IsDeleted bool `bson:"is_deleted" json:"is_deleted"` + IsArchived bool `bson:"is_archived" json:"is_archived"` + AgentID string `bson:"agent_id" json:"agent_id"` + // is allowed to run multiple times + MultiRun bool `bson:"multi_run" json:"multi_run"` + // target is container name when k8s, service name when pm + Target string `bson:"target,omitempty" json:"target"` + // generate SubTasks with predefine build module, + // query filter param: ServiceTmpl, BuildModuleVer + // if nil，use pipeline self define SubTasks + BuildModuleVer string `bson:"build_module_ver,omitempty" json:"build_module_ver"` + ServiceName string `bson:"service_name,omitempty" json:"service_name,omitempty"` + // TaskArgs single workflow args + TaskArgs *models.TaskArgs `bson:"task_args,omitempty" json:"task_args,omitempty"` + // WorkflowArgs multi workflow args + WorkflowArgs *models.WorkflowTaskArgs `bson:"workflow_args" json:"workflow_args,omitempty"` + // TestArgs test workflow args + TestArgs *models.TestTaskArgs `bson:"test_args,omitempty" json:"test_args,omitempty"` + // ServiceTaskArgs sh deploy args + ServiceTaskArgs *models.ServiceTaskArgs `bson:"service_args,omitempty" json:"service_args,omitempty"` + // ArtifactPackageTaskArgs arguments for artifact-package type tasks + ArtifactPackageTaskArgs *models.ArtifactPackageTaskArgs `bson:"artifact_package_args,omitempty" json:"artifact_package_args,omitempty"` + // ConfigPayload system config info + ConfigPayload *models.ConfigPayload `bson:"configpayload" json:"config_payload,omitempty"` + Error string `bson:"error,omitempty" json:"error,omitempty"` + Services [][]*models.ProductService `bson:"services" json:"services"` + Render *models.RenderInfo `bson:"render" json:"render"` + StorageURI string `bson:"storage_uri,omitempty" json:"storage_uri,omitempty"` + // interface{} 为types.TestReport + TestReports map[string]interface{} `bson:"test_reports,omitempty" json:"test_reports,omitempty"` + RwLock sync.Mutex `bson:"-" json:"-"` + ResetImage bool `bson:"resetImage" json:"resetImage"` + ResetImagePolicy setting.ResetImagePolicyType `bson:"reset_image_policy" json:"reset_image_policy"` + TriggerBy *models.TriggerBy `bson:"trigger_by,omitempty" json:"trigger_by,omitempty"` + Features []string `bson:"features" json:"features"` + IsRestart bool `bson:"is_restart" json:"is_restart"` + StorageEndpoint string `bson:"storage_endpoint" json:"storage_endpoint"` + Releases []Release `json:"releases"` +} + +type Release struct { + ID primitive.ObjectID `json:"id"` + Version string `json:"version"` +} diff --git a/pkg/types/env.go b/pkg/types/env.go new file mode 100644 index 0000000000000000000000000000000000000000..8b75d7ea2cc452e6a06ca81505a7932d1b860a28 --- /dev/null +++ b/pkg/types/env.go @@ -0,0 +1,26 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +type EnvType string + +const ( + GeneralEnv EnvType = "general" + ShareEnv EnvType = "share" +) + +const DebugImage = "koderover.tencentcloudcr.com/koderover-public/zadig-debug:v0.1.0" diff --git a/pkg/types/git.go b/pkg/types/git.go new file mode 100644 index 0000000000000000000000000000000000000000..e19eda34e71c03cd3abcbb7683b355cd7f8405e8 --- /dev/null +++ b/pkg/types/git.go @@ -0,0 +1,24 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +type AuthType string + +const ( + SSHAuthType AuthType = "SSH" + PrivateAccessTokenAuthType AuthType = "PrivateAccessToken" +) diff --git a/pkg/types/ide.go b/pkg/types/ide.go new file mode 100644 index 0000000000000000000000000000000000000000..c4a508811260e8a6ef857780b3d602fbf8cc3f68 --- /dev/null +++ b/pkg/types/ide.go @@ -0,0 +1,33 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +type WorkloadInfo struct { + PodName string `json:"pod_name"` + PodNamespace string `json:"pod_namespace"` +} + +const IDEContainerNameDev = "dev" +const IDEContainerNameSidecar = "sidecar" + +type StartDevmodeInfo struct { + DevImage string `json:"dev_image"` +} + +const IDESidecarImage = "koderover.tencentcloudcr.com/koderover-public/zgctl-sidecar:20220526172433-amd64" + +const DevmodeWorkDir = "/home/zadig/" diff --git a/pkg/types/job.go b/pkg/types/job.go new file mode 100644 index 0000000000000000000000000000000000000000..37604362eb3abc19cbee542aeadffb6b76168975 --- /dev/null +++ b/pkg/types/job.go @@ -0,0 +1,24 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +type JobStatus string + +const ( + JobSuccess JobStatus = "success" + JobFail JobStatus = "fail" +) diff --git a/pkg/types/job/job.go b/pkg/types/job/job.go new file mode 100644 index 0000000000000000000000000000000000000000..d56c57868ada9fdab282322c158876d194142540 --- /dev/null +++ b/pkg/types/job/job.go @@ -0,0 +1,27 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package job + +const ( + JobOutputDir = "/zadig/results/" + JobTerminationFile = "/zadig/termination" +) + +type JobOutput struct { + Name string `json:"name"` + Value string `json:"value"` +} diff --git a/pkg/types/labels.go b/pkg/types/labels.go new file mode 100644 index 0000000000000000000000000000000000000000..c9d81561b4d135bb403a9e97bd436f43016928c7 --- /dev/null +++ b/pkg/types/labels.go @@ -0,0 +1,29 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import "fmt" + +const ZadigDomain = "zadig.koderover.com" +const Zadig = "zadig" + +var ZadigLabelKeyGlobalOwner = fmt.Sprintf("%s/owner", ZadigDomain) + +const IstioLabelKeyInjection = "istio-injection" +const IstioLabelValueInjection = "enabled" + +var OriginSpec = fmt.Sprintf("%s/origin", ZadigDomain) diff --git a/pkg/types/object_storage.go b/pkg/types/object_storage.go new file mode 100644 index 0000000000000000000000000000000000000000..b162ca714371bc168c9cfb0ec8f342e32abef26a --- /dev/null +++ b/pkg/types/object_storage.go @@ -0,0 +1,32 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +type ObjectStorageInfo struct { + Endpoint string `bson:"endpoint" json:"endpoint" yaml:"endpoint"` + AK string `bson:"AK" json:"AK" yaml:"AK"` + SK string `bson:"SK" json:"SK" yaml:"SK"` + Bucket string `bson:"bucket" json:"bucket" yaml:"bucket"` + Insecure bool `bson:"insecure" json:"insecure" yaml:"insecure"` + Provider int8 `bson:"provider" json:"provider" yaml:"provider"` +} + +type ObjectStoragePathDetail struct { + FilePath string `bson:"file_path" json:"file_path" yaml:"file_path"` + AbsFilePath string `bson:"abs_file_path" json:"abs_file_path" yaml:"abs_file_path"` + DestinationPath string `bson:"dest_path" json:"dest_path" yaml:"dest_path"` +} diff --git a/pkg/types/permission/permission.go b/pkg/types/permission/permission.go deleted file mode 100644 index abbbfaf7af2774b26e646357c567f7404845f020..0000000000000000000000000000000000000000 --- a/pkg/types/permission/permission.go +++ /dev/null @@ -1,66 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package permission - -const ( - SuperUserUUID = "00000" //超级用户权限 - //产品工作流相关 - WorkflowTaskUUID = "30001" //产品工作流(执行任务、重启任务、取消任务) - WorkflowUpdateUUID = "30002" //产品工作流(更新) - WorkflowCreateUUID = "30003" //产品工作流(创建) - WorkflowDeleteUUID = "30004" //产品工作流(删除) - WorkflowListUUID = "30005" //产品工作流(查看) - WorkflowDeliveryUUID = "30006" //产品工作流(添加交付) - - //集成环境(测试环境) - TestEnvCreateUUID = "40001" //创建环境 - TestEnvDeleteUUID = "40002" //删除环境 - TestEnvManageUUID = "40003" //环境管理(环境更新、单服务更新、服务伸缩、服务重启、configmap更新、configmap回滚、重启实例、yaml导出) - TestEnvListUUID = "40004" //环境查看 - TestEnvShareUUID = "40005" //环境授权 - TestUpdateEnvUUID = "40010" //更新环境 - - //集成环境(生产环境) - ProdEnvListUUID = "40006" //环境查看 - ProdEnvCreateUUID = "40007" //创建环境 - ProdEnvManageUUID = "40008" //环境管理(环境更新、单服务更新、服务伸缩、服务重启、configMap更新、configMap回滚、重启实例、yaml导出) - ProdEnvDeleteUUID = "40009" //删除环境 - - //原项目管理相关 - //构建管理 - BuildDeleteUUID = "50007" //删除构建配置 - BuildManageUUID = "50008" //构建配置管理(创建构建、修改构建、修改服务组件) - BuildListUUID = "50019" //构建配置查看 - - //服务管理 - ServiceTemplateEditUUID = "50022" //修改服务编排 - ServiceTemplateManageUUID = "50023" //服务模板管理(创建服务模板、修改服务模板) - ServiceTemplateDeleteUUID = "50024" //删除服务模板 - ServiceTemplateListUUID = "50025" //查看服务模板 - - //测试管理 - TestDeleteUUID = "50010" //删除测试 - TestManageUUID = "50011" //测试管理(创建测试、修改测试) - TestListUUID = "50020" //查看测试 - - //交付中心 - ReleaseDeleteUUID = "70001" //交付中心删除版本 - - ParamType = 1 - QueryType = 2 - ContextKeyType = 3 -) diff --git a/pkg/types/permission/user.go b/pkg/types/permission/user.go deleted file mode 100644 index 24b3b5f6a820ebb4fb41462008977e48aeedfd44..0000000000000000000000000000000000000000 --- a/pkg/types/permission/user.go +++ /dev/null @@ -1,40 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package permission - -const ( - AnonymousUserID = 0 - AnonymousUserName = "Anonymous" -) - -type User struct { - ID int `json:"id"` - Name string `json:"name"` - Email string `json:"email"` - Password string `json:"password"` - Phone string `json:"phone"` - IsAdmin bool `json:"isAdmin"` - IsSuperUser bool `json:"isSuperUser"` - IsTeamLeader bool `json:"isTeamLeader"` - OrganizationID int `json:"organization_id"` - Directory string `json:"directory"` - LastLoginAt int64 `json:"lastLogin"` - CreatedAt int64 `json:"created_at"` - UpdatedAt int64 `json:"updated_at"` -} - -var AnonymousUser = &User{ID: AnonymousUserID, Name: AnonymousUserName} diff --git a/pkg/types/policy.go b/pkg/types/policy.go new file mode 100644 index 0000000000000000000000000000000000000000..bda3cd675fd7afaf4963a26a9880d26ef674fde2 --- /dev/null +++ b/pkg/types/policy.go @@ -0,0 +1,77 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +//PolicyMetaScope resource scope for permission +type PolicyMetaScope string + +const ( + SystemScope PolicyMetaScope = "system" + ProjectScope PolicyMetaScope = "project" +) + +type Policy struct { + Name string `json:"name"` + Description string `json:"description"` + UpdateTime int64 `json:"update_time"` + Rules []*Rule `json:"rules"` +} + +type Rule struct { + Verbs []string `json:"verbs"` + Resources []string `json:"resources"` + Kind string `json:"kind"` + MatchAttributes []MatchAttribute `json:"match_attributes"` +} + +type MatchAttribute struct { + Key string `json:"key"` + Value string `json:"value"` +} + +type PolicyMeta struct { + Resource string `json:"resource"` + Alias string `json:"alias"` + Description string `json:"description"` + Rules []*RuleMeta `json:"rules"` +} + +type RuleMeta struct { + Action string `json:"action"` + Alias string `json:"alias"` + Description string `json:"description"` + Rules []*ActionRule `json:"rules"` +} + +type ActionRule struct { + Method string `json:"method"` + Endpoint string `json:"endpoint"` + ResourceType string `json:"resourceType,omitempty"` + IDRegex string `json:"idRegex,omitempty"` + MatchAttributes []*Attribute `json:"matchAttributes,omitempty"` + Filter bool `json:"filter,omitempty"` +} + +type Attribute struct { + Key string `json:"key"` + Value string `json:"value"` +} + +type PolicyRule struct { + Methods []string `json:"methods"` + Endpoint string `json:"endpoint"` +} diff --git a/pkg/types/private_key.go b/pkg/types/private_key.go new file mode 100644 index 0000000000000000000000000000000000000000..54e07563e199e4ad61d4323956c2911fba6a675d --- /dev/null +++ b/pkg/types/private_key.go @@ -0,0 +1,36 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +type Probe struct { + ProbeScheme string `bson:"probe_type" json:"probe_type"` + HttpProbe *HTTPGetAction `bson:"http_probe" json:"http_probe"` +} + +type HTTPHeader struct { + Name string `bson:"name" json:"name"` + Value string `bson:"value" json:"value"` +} + +type HTTPGetAction struct { + Path string `bson:"path" json:"path"` + Port int `bson:"port" json:"port"` + Host string `bson:"-" json:"host,omitempty"` + HTTPHeaders []*HTTPHeader `bson:"http_headers" json:"http_headers"` + TimeOutSecond int `bson:"timeout_second" json:"timeout_second"` + ResponseSuccessFlag string `bson:"response_success_flag" json:"response_success_flag"` +} diff --git a/pkg/types/reaper.go b/pkg/types/reaper.go new file mode 100644 index 0000000000000000000000000000000000000000..7e3fed0a8894f8d9832fc4c0608e89253e2b9efc --- /dev/null +++ b/pkg/types/reaper.go @@ -0,0 +1,29 @@ +/* + Copyright 2022 The KodeRover Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package types + +type ReaperType string + +const ( + BuildReaperType ReaperType = "Build" + TestReaperType ReaperType = "Test" + ScanningReaperType ReaperType = "Scanning" +) + +const ( + ScanningTypeSonar = "sonarQube" +) diff --git a/pkg/types/repo.go b/pkg/types/repo.go index 24cf3b6b6d2c1cada7b6c9a9c9d63ef0d3c64574..b8430a613796e254fdee0a70cd5865b42bb8d226 100644 --- a/pkg/types/repo.go +++ b/pkg/types/repo.go @@ -19,39 +19,65 @@ package types import ( "fmt" "regexp" + "strings" "time" ) // Repository struct type Repository struct { - // Source is github, gitlab - Source string `bson:"source,omitempty" json:"source,omitempty"` - RepoOwner string `bson:"repo_owner" json:"repo_owner"` - RepoName string `bson:"repo_name" json:"repo_name"` - RemoteName string `bson:"remote_name,omitempty" json:"remote_name,omitempty"` - Branch string `bson:"branch" json:"branch"` - PR int `bson:"pr,omitempty" json:"pr,omitempty"` - Tag string `bson:"tag,omitempty" json:"tag,omitempty"` - CommitID string `bson:"commit_id,omitempty" json:"commit_id,omitempty"` - CommitMessage string `bson:"commit_message,omitempty" json:"commit_message,omitempty"` - CheckoutPath string `bson:"checkout_path,omitempty" json:"checkout_path,omitempty"` - SubModules bool `bson:"submodules,omitempty" json:"submodules,omitempty"` + Source string `bson:"source,omitempty" json:"source,omitempty" yaml:"source,omitempty"` + RepoOwner string `bson:"repo_owner" json:"repo_owner" yaml:"repo_owner"` + RepoNamespace string `bson:"repo_namespace" json:"repo_namespace" yaml:"repo_namespace"` + RepoName string `bson:"repo_name" json:"repo_name" yaml:"repo_name"` + RemoteName string `bson:"remote_name,omitempty" json:"remote_name,omitempty" yaml:"remote_name,omitempty"` + Branch string `bson:"branch" json:"branch" yaml:"branch"` + PR int `bson:"pr,omitempty" json:"pr,omitempty" yaml:"pr,omitempty"` + Tag string `bson:"tag,omitempty" json:"tag,omitempty" yaml:"tag,omitempty"` + CommitID string `bson:"commit_id,omitempty" json:"commit_id,omitempty" yaml:"commit_id,omitempty"` + CommitMessage string `bson:"commit_message,omitempty" json:"commit_message,omitempty" yaml:"commit_message,omitempty"` + CheckoutPath string `bson:"checkout_path,omitempty" json:"checkout_path,omitempty" yaml:"checkout_path,omitempty"` + SubModules bool `bson:"submodules,omitempty" json:"submodules,omitempty" yaml:"submodules,omitempty"` // UseDefault defines if the repo can be configured in start pipeline task page - UseDefault bool `bson:"use_default,omitempty" json:"use_default,omitempty"` + UseDefault bool `bson:"use_default,omitempty" json:"use_default,omitempty" yaml:"use_default,omitempty"` // IsPrimary used to generated image and package name, each build has one primary repo - IsPrimary bool `bson:"is_primary" json:"is_primary"` - CodehostID int `bson:"codehost_id" json:"codehost_id"` + IsPrimary bool `bson:"is_primary" json:"is_primary" yaml:"is_primary"` + CodehostID int `bson:"codehost_id" json:"codehost_id" yaml:"codehost_id"` // add - OauthToken string `bson:"oauth_token" json:"oauth_token"` - Address string `bson:"address" json:"address"` - AuthorName string `bson:"author_name,omitempty" json:"author_name,omitempty"` - CheckoutRef string `bson:"checkout_ref,omitempty" json:"checkout_ref,omitempty"` + OauthToken string `bson:"oauth_token" json:"oauth_token" yaml:"oauth_token"` + Address string `bson:"address" json:"address" yaml:"address"` + AuthorName string `bson:"author_name,omitempty" json:"author_name,omitempty" yaml:"author_name,omitempty"` + CheckoutRef string `bson:"checkout_ref,omitempty" json:"checkout_ref,omitempty" yaml:"checkout_ref,omitempty"` // codehub - ProjectUUID string `bson:"project_uuid,omitempty" json:"project_uuid,omitempty"` - RepoUUID string `bson:"repo_uuid,omitempty" json:"repo_uuid,omitempty"` - RepoID string `bson:"repo_id,omitempty" json:"repo_id,omitempty"` - Username string `bson:"username,omitempty" json:"username,omitempty"` - Password string `bson:"password,omitempty" json:"password,omitempty"` + ProjectUUID string `bson:"project_uuid,omitempty" json:"project_uuid,omitempty" yaml:"project_uuid,omitempty"` + RepoUUID string `bson:"repo_uuid,omitempty" json:"repo_uuid,omitempty" yaml:"repo_uuid,omitempty"` + RepoID string `bson:"repo_id,omitempty" json:"repo_id,omitempty" yaml:"repo_id,omitempty"` + Username string `bson:"username,omitempty" json:"username,omitempty" yaml:"username,omitempty"` + Password string `bson:"password,omitempty" json:"password,omitempty" yaml:"password,omitempty"` + // Now EnableProxy is not something we store. We decide this on runtime + EnableProxy bool `bson:"-" json:"enable_proxy,omitempty" yaml:"enable_proxy,omitempty"` + // FilterRegexp is the regular expression filter for the branches and tags + FilterRegexp string `bson:"-" json:"filter_regexp,omitempty" yaml:"filter_regexp,omitempty"` + // The address of the code base input of the other type + AuthType AuthType `bson:"auth_type,omitempty" json:"auth_type,omitempty" yaml:"auth_type,omitempty"` + SSHKey string `bson:"ssh_key,omitempty" json:"ssh_key,omitempty" yaml:"ssh_key,omitempty"` + PrivateAccessToken string `bson:"private_access_token,omitempty" json:"private_access_token,omitempty" yaml:"private_access_token,omitempty"` +} + +type BranchFilterInfo struct { + // repository identifier + CodehostID int `bson:"codehost_id" json:"codehost_id"` + RepoOwner string `bson:"repo_owner" json:"repo_owner"` + RepoName string `bson:"repo_name" json:"repo_name"` + RepoNamespace string `bson:"repo_namespace" json:"repo_namespace"` + // actual regular expression filter + FilterRegExp string `bson:"filter_regexp" json:"filter_regexp"` +} + +func (bf *BranchFilterInfo) GetNamespace() string { + if len(bf.RepoNamespace) > 0 { + return bf.RepoNamespace + } + return bf.RepoOwner } // GetReleaseCandidateTag 返回待发布对象Tag @@ -86,3 +112,71 @@ func (repo *Repository) GetReleaseCandidateTag(taskID int64) string { } return tag } + +func (repo *Repository) GetRepoNamespace() string { + if repo.RepoNamespace != "" { + return repo.RepoNamespace + } + return repo.RepoOwner +} + +const ( + // ProviderGithub ... + ProviderGithub = "github" + // ProviderGitlab ... + ProviderGitlab = "gitlab" + + // ProviderGerrit + ProviderGerrit = "gerrit" + + // ProviderCodehub + ProviderCodehub = "codehub" + + // ProviderGitee + ProviderGitee = "gitee" + + // ProviderOther + ProviderOther = "other" +) + +// PRRef returns refs format +// It will check repo provider type, by default returns github refs format. +// +// e.g. github returns refs/pull/1/head +// e.g. gitlab returns merge-requests/1/head +func (r *Repository) PRRef() string { + if strings.ToLower(r.Source) == ProviderGitlab || strings.ToLower(r.Source) == ProviderCodehub { + return fmt.Sprintf("merge-requests/%d/head", r.PR) + } else if strings.ToLower(r.Source) == ProviderGerrit { + return r.CheckoutRef + } + return fmt.Sprintf("refs/pull/%d/head", r.PR) +} + +// BranchRef returns branch refs format +// e.g. refs/heads/master +func (r *Repository) BranchRef() string { + return fmt.Sprintf("refs/heads/%s", r.Branch) +} + +// TagRef returns the tag ref of current repo +// e.g. refs/tags/v1.0.0 +func (r *Repository) TagRef() string { + return fmt.Sprintf("refs/tags/%s", r.Tag) +} + +// Ref returns the changes ref of current repo in the following order: +// 1. tag ref +// 2. branch ref +// 3. pr ref +func (r *Repository) Ref() string { + if len(r.Tag) > 0 { + return r.TagRef() + } else if len(r.Branch) > 0 { + return r.BranchRef() + } else if r.PR > 0 { + return r.PRRef() + } + + return "" +} diff --git a/pkg/types/scanning.go b/pkg/types/scanning.go new file mode 100644 index 0000000000000000000000000000000000000000..30f1a183f122ab6d16c3403d2b60576ce656ac77 --- /dev/null +++ b/pkg/types/scanning.go @@ -0,0 +1,50 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import ( + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/setting" +) + +type ScanningAdvancedSetting struct { + ClusterID string `bson:"cluster_id" json:"cluster_id"` + Timeout int64 `bson:"timeout" json:"timeout"` + ResReq setting.Request `bson:"res_req" json:"res_req"` + ResReqSpec setting.RequestSpec `bson:"res_req_spec" json:"res_req_spec"` + HookCtl *ScanningHookCtl `bson:"hook_ctl" json:"hook_ctl"` +} + +type ScanningHookCtl struct { + Enabled bool `bson:"enabled" json:"enabled"` + Items []*ScanningHook `bson:"items" json:"items"` +} + +type ScanningHook struct { + CodehostID int `bson:"codehost_id" json:"codehost_id"` + Source string `bson:"source" json:"source"` + RepoOwner string `bson:"repo_owner" json:"repo_owner"` + RepoName string `bson:"repo_name" json:"repo_name"` + Branch string `bson:"branch" json:"branch"` + Events []config.HookEventType `bson:"events" json:"events"` + MatchFolders []string `bson:"match_folders" json:"match_folders"` +} + +type SonarInfo struct { + ServerAddress string `bson:"server_address" json:"server_address"` + Token string `bson:"token" json:"token"` +} diff --git a/pkg/types/step/common.go b/pkg/types/step/common.go new file mode 100644 index 0000000000000000000000000000000000000000..5ccc0d7559a7182ad65adc5995b405487e4e6f09 --- /dev/null +++ b/pkg/types/step/common.go @@ -0,0 +1,39 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package step + +type Proxy struct { + Type string `bson:"type" json:"type" yaml:"type"` + Address string `bson:"address" json:"address" yaml:"address"` + Port int `bson:"port" json:"port" yaml:"port"` + NeedPassword bool `bson:"need_password" json:"need_password" yaml:"need_password"` + Username string `bson:"username" json:"username" yaml:"username"` + Password string `bson:"password" json:"password" yaml:"password"` + EnableRepoProxy bool `bson:"enable_repo_proxy" json:"enable_repo_proxy" yaml:"enable_repo_proxy"` + EnableApplicationProxy bool `bson:"enable_application_proxy" json:"enable_application_proxy" yaml:"enable_application_proxy"` +} + +type S3 struct { + Ak string `bson:"ak" json:"ak" yaml:"ak"` + Sk string `bson:"sk" json:"sk" yaml:"sk"` + Endpoint string `bson:"endpoint" json:"endpoint" yaml:"endpoint"` + Bucket string `bson:"bucket" json:"bucket" yaml:"bucket"` + Subfolder string `bson:"subfolder" json:"subfolder" yaml:"subfolder"` + Insecure bool `bson:"insecure" json:"insecure" yaml:"insecure"` + Provider int8 `bson:"provider" json:"provider" yaml:"provider"` + Protocol string `bson:"protocol" json:"protocol" yaml:"protocol"` +} diff --git a/pkg/types/step/step_archive.go b/pkg/types/step/step_archive.go new file mode 100644 index 0000000000000000000000000000000000000000..0d8113a013ae3196dfd4500ab1d5b52609a28ae5 --- /dev/null +++ b/pkg/types/step/step_archive.go @@ -0,0 +1,29 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package step + +type StepArchiveSpec struct { + UploadDetail []*Upload `bson:"upload_detail" json:"upload_detail" yaml:"upload_detail"` + ObjectStorageID string `bson:"object_storage_id" json:"object_storage_id" yaml:"object_storage_id"` + S3 *S3 `bson:"s3_storage" json:"s3_storage" yaml:"s3_storage"` +} + +type Upload struct { + FilePath string `bson:"file_path" json:"file_path" yaml:"file_path"` + AbsFilePath string `bson:"abs_file_path" json:"aabs_file_pathk" yaml:"abs_file_path"` + DestinationPath string `bson:"dest_path" json:"dest_path" yaml:"dest_path"` +} diff --git a/pkg/types/step/step_deploy.go b/pkg/types/step/step_deploy.go new file mode 100644 index 0000000000000000000000000000000000000000..6943d19b42f0cac98f69ff9a9625f4b1ec2fbd63 --- /dev/null +++ b/pkg/types/step/step_deploy.go @@ -0,0 +1,35 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package step + +type StepDeploySpec struct { + Env string `bson:"env" json:"env" yaml:"env"` + ServiceName string `bson:"service_name" json:"service_name" yaml:"service_name"` + ServiceType string `bson:"service_type" json:"service_type" yaml:"service_type"` + ServiceModule string `bson:"service_module" json:"service_module" yaml:"service_module"` + Image string `bson:"image" json:"image" yaml:"image"` + ClusterID string `bson:"cluster_id" json:"cluster_id" yaml:"cluster_id"` + Timeout int `bson:"timeout" json:"timeout" yaml:"timeout"` + ReplaceResources []Resource `bson:"replace_resources" json:"replace_resources" yaml:"replace_resources"` +} + +type Resource struct { + Name string `bson:"name" json:"name" yaml:"name"` + Kind string `bson:"kind" json:"kind" yaml:"kind"` + Container string `bson:"container" json:"container" yaml:"container"` + Origin string `bson:"origin" json:"origin" yaml:"origin"` +} diff --git a/pkg/types/step/step_docker_build.go b/pkg/types/step/step_docker_build.go new file mode 100644 index 0000000000000000000000000000000000000000..065a1c459a717019d07d40e902590629deac6aa2 --- /dev/null +++ b/pkg/types/step/step_docker_build.go @@ -0,0 +1,56 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package step + +import ( + "fmt" + + "github.com/koderover/zadig/pkg/setting" +) + +type StepDockerBuildSpec struct { + Source string `bson:"source" json:"source" yaml:"source"` + WorkDir string `bson:"work_dir" json:"work_dir" yaml:"work_dir"` + RegistryHost string `bson:"registry_host" json:"registry_host" yaml:"registry_host"` + DockerFile string `bson:"docker_file" json:"docker_file" yaml:"docker_file"` + ImageName string `bson:"image_name" json:"image_name" yaml:"image_name"` + BuildArgs string `bson:"build_args" json:"build_args" yaml:"build_args"` + ImageReleaseTag string `bson:"image_release_tag" json:"image_release_tag" yaml:"image_release_tag"` + DockerTemplateContent string `bson:"docker_template_content" json:"docker_template_content" yaml:"docker_template_content"` + Proxy *Proxy `bson:"proxy" json:"proxy" yaml:"proxy"` + IgnoreCache bool `bson:"ignore_cache" json:"ignore_cache" yaml:"ignore_cache"` + DockerRegistry *DockerRegistry `bson:"docker_registry" json:"docker_registry" yaml:"docker_registry"` +} + +type DockerRegistry struct { + DockerRegistryID string `bson:"docker_registry_id" json:"docker_registry_id" yaml:"docker_registry_id"` + Host string `bson:"host" json:"host" yaml:"host"` + Namespace string `bson:"namespace" json:"namespace" yaml:"namespace"` + UserName string `bson:"username" json:"username" yaml:"username"` + Password string `bson:"password" json:"password" yaml:"password"` +} + +func (s *StepDockerBuildSpec) GetDockerFile() string { + // if the source of the dockerfile is from template, we write our own dockerfile + if s.Source == setting.DockerfileSourceTemplate { + return fmt.Sprintf("/%s", setting.ZadigDockerfilePath) + } + if s.DockerFile == "" { + return "Dockerfile" + } + return s.DockerFile +} diff --git a/pkg/types/step/step_git.go b/pkg/types/step/step_git.go new file mode 100644 index 0000000000000000000000000000000000000000..cdbc83ab05c3070093ec3360529041126af90579 --- /dev/null +++ b/pkg/types/step/step_git.go @@ -0,0 +1,56 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package step + +import ( + "fmt" + + "github.com/koderover/zadig/pkg/types" +) + +type StepGitSpec struct { + Repos []*types.Repository `bson:"repos" json:"repos" yaml:"repos"` + Proxy *Proxy `bson:"proxy" json:"proxy" yaml:"proxy"` +} + +const ( + // Oauth prefix + OauthTokenPrefix = "oauth2" + + FileName = "reaper.tar.gz" +) + +func (p *Proxy) GetProxyURL() string { + var uri string + if p.NeedPassword { + uri = fmt.Sprintf("%s://%s:%s@%s:%d", + p.Type, + p.Username, + p.Password, + p.Address, + p.Port, + ) + return uri + } + + uri = fmt.Sprintf("%s://%s:%d", + p.Type, + p.Address, + p.Port, + ) + return uri +} diff --git a/pkg/types/step/step_helm_deploy.go b/pkg/types/step/step_helm_deploy.go new file mode 100644 index 0000000000000000000000000000000000000000..04c25ae2c7d7981e4d67c966c3454daeebcece09 --- /dev/null +++ b/pkg/types/step/step_helm_deploy.go @@ -0,0 +1,33 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package step + +type StepHelmDeploySpec struct { + Env string `bson:"env" json:"env" yaml:"env"` + ServiceName string `bson:"service_name" json:"service_name" yaml:"service_name"` + ServiceType string `bson:"service_type" json:"service_type" yaml:"service_type"` + ImageAndModules []*ImageAndServiceModule `bson:"image_and_service_modules" json:"image_and_service_modules" yaml:"image_and_service_modules"` + ClusterID string `bson:"cluster_id" json:"cluster_id" yaml:"cluster_id"` + ReleaseName string `bson:"release_name" json:"release_name" yaml:"release_name"` + Timeout int `bson:"timeout" json:"timeout" yaml:"timeout"` + ReplaceResources []Resource `bson:"replace_resources" json:"replace_resources" yaml:"replace_resources"` +} + +type ImageAndServiceModule struct { + ServiceModule string `bson:"service_module" json:"service_module" yaml:"service_module"` + Image string `bson:"image" json:"image" yaml:"image"` +} diff --git a/pkg/types/step/step_shell.go b/pkg/types/step/step_shell.go new file mode 100644 index 0000000000000000000000000000000000000000..6ee3cb074020a93f7ce101e5e34e41a2669777d3 --- /dev/null +++ b/pkg/types/step/step_shell.go @@ -0,0 +1,22 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package step + +type StepShellSpec struct { + Scripts []string `bson:"scripts" json:"scripts" yaml:"scripts,omitempty"` + Script string `bson:"script" json:"script" yaml:"script"` +} diff --git a/pkg/types/step/step_tool_install.go b/pkg/types/step/step_tool_install.go new file mode 100644 index 0000000000000000000000000000000000000000..bd39c7a58829e89f5f09783581ff2939b3515436 --- /dev/null +++ b/pkg/types/step/step_tool_install.go @@ -0,0 +1,15 @@ +package step + +type StepToolInstallSpec struct { + Installs []*Tool `bson:"installs" json:"installs" yaml:"installs"` + S3Storage *S3 `bson:"s3_storage" json:"s3_storage" yaml:"s3_storage"` +} + +type Tool struct { + Name string `bson:"name" json:"name" yaml:"name"` + Version string `bson:"version" json:"version" yaml:"version"` + Scripts []string `bson:"scripts" json:"scripts" yaml:"scripts"` + BinPath string `bson:"bin_path" json:"bin_path" yaml:"bin_path"` + Envs []string `bson:"envs" json:"envs" yaml:"envs"` + Download string `bson:"download" json:"download" yaml:"download"` +} diff --git a/pkg/types/user.go b/pkg/types/user.go new file mode 100644 index 0000000000000000000000000000000000000000..3eab93d4550849016400d39e9726c71ec72baf34 --- /dev/null +++ b/pkg/types/user.go @@ -0,0 +1,52 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import ( + "github.com/koderover/zadig/pkg/setting" +) + +type UserInfo struct { + LastLoginTime int64 `json:"last_login_time"` + Uid string `json:"uid"` + Name string `json:"name"` + IdentityType string `gorm:"default:'unknown'" json:"identity_type"` + Email string `json:"email"` + Phone string `json:"phone"` + Account string `json:"account"` + APIToken string `json:"token"` + SystemRoleBindings []*RoleBinding `json:"system_role_bindings"` + Admin bool `json:"admin"` +} + +type UsersResp struct { + Users []UserInfo `json:"users"` + TotalCount int64 `json:"total_count"` +} + +type RoleBinding struct { + Name string `json:"name"` + UID string `json:"uid"` + Role string `json:"role"` + Preset bool `json:"preset"` + Type setting.ResourceType `json:"type"` +} + +type UserCountByType struct { + IdentityType string `gorm:"default:'unknown'" json:"identity_type" gorm:"identity_type"` + Count int64 `json:"count" gorm:"count"` +} diff --git a/pkg/types/workload.go b/pkg/types/workload.go new file mode 100644 index 0000000000000000000000000000000000000000..2b714c61c02da42ed897a1781ed5ef2b54fe9d71 --- /dev/null +++ b/pkg/types/workload.go @@ -0,0 +1,37 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import corev1 "k8s.io/api/core/v1" + +type WorkloadType string + +const ( + DeploymentWorkload WorkloadType = "deployment" + StatefulSetWorkload WorkloadType = "statefulset" +) + +type DindPod struct { + ClusterID string + ClusterName string + Pod *corev1.Pod +} + +const DindStatefulSetName = "dind" +const DindContainerName = "dind" +const DindMountName = "zadig-docker" +const DindMountPath = "/var/lib/docker" diff --git a/pkg/util/array.go b/pkg/util/array.go new file mode 100644 index 0000000000000000000000000000000000000000..dba055642a83c29c8a1b7732829e88b741d026b9 --- /dev/null +++ b/pkg/util/array.go @@ -0,0 +1,26 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +func InStringArray(target string, str_array []string) bool { + for _, element := range str_array { + if target == element { + return true + } + } + return false +} diff --git a/pkg/util/deepcopy/deepcopy.go b/pkg/util/deepcopy/deepcopy.go new file mode 100644 index 0000000000000000000000000000000000000000..bc41cd1f2e80ab287d5ec646ebe895e46eea8ad8 --- /dev/null +++ b/pkg/util/deepcopy/deepcopy.go @@ -0,0 +1,145 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package deepcopy + +import ( + "fmt" + "reflect" + "unsafe" +) + +// FromTo deep copies original and assigns the copy to the copy argument (pointer). +func FromTo(original, copy interface{}) error { + if original == nil { + copy = nil + return nil + } else if copy == nil { // TODO try to initialize it here + return fmt.Errorf("FromTo: copy target is nil, it should be a valid pointer") + // copyValue := reflect.New(value.Type().Elem()).Elem() + // copy = copyValue.Interface() + } + copyValue := reflect.ValueOf(copy) + if copyValue.Kind() != reflect.Ptr { + return fmt.Errorf("FromTo: copy target type %T and not a pointer", copy) + } + value := reflect.ValueOf(original) + if value.Kind() == reflect.Ptr { + if value.IsNil() { + copy = nil // TODO return typed nil + return nil + } + value = value.Elem() + } + copyValue.Elem().Set(deepCopy(value)) + return nil +} + +func deepCopy(original reflect.Value) reflect.Value { + switch original.Kind() { + case reflect.Slice: + return deepCopySlice(original) + case reflect.Map: + return deepCopyMap(original) + case reflect.Ptr: + return deepCopyPointer(original) + case reflect.Struct: + return deepCopyStruct(original) + case reflect.Chan: + return deepCopyChan(original) + case reflect.Array: + return deepCopyArray(original) + default: + return forceCopyValue(original) + } +} + +func deepCopySlice(original reflect.Value) reflect.Value { + if original.IsNil() { + return original + } + copy := reflect.MakeSlice(original.Type(), 0, 0) + for i := 0; i < original.Len(); i++ { + elementCopy := deepCopy(original.Index(i)) + copy = reflect.Append(copy, elementCopy) + } + return copy +} + +func deepCopyArray(original reflect.Value) reflect.Value { + if original.Len() == 0 { + // it cannot be changed anyway, so we can return the original + return original + } + elementType := original.Index(0).Type() + arrayType := reflect.ArrayOf(original.Len(), elementType) + newPointer := reflect.New(arrayType) + copy := newPointer.Elem() + for i := 0; i < original.Len(); i++ { + subCopy := deepCopy(original.Index(i)) + copy.Index(i).Set(subCopy) + } + return copy +} + +func deepCopyMap(original reflect.Value) reflect.Value { + if original.IsNil() { + return original + } + keyType := original.Type().Key() + valueType := original.Type().Elem() + mapType := reflect.MapOf(keyType, valueType) + copy := reflect.MakeMap(mapType) + for _, key := range original.MapKeys() { + value := deepCopy(original.MapIndex(key)) + copy.SetMapIndex(key, value) + } + return copy +} + +func deepCopyPointer(original reflect.Value) reflect.Value { + if original.IsNil() { + return original + } + element := original.Elem() + copy := reflect.New(element.Type()) + copyElement := deepCopy(element) + copy.Elem().Set(copyElement) + return copy +} + +func deepCopyStruct(original reflect.Value) reflect.Value { + copy := reflect.New(original.Type()).Elem() + copy.Set(original) + for i := 0; i < original.NumField(); i++ { + fieldValue := copy.Field(i) + fieldValue = reflect.NewAt(fieldValue.Type(), unsafe.Pointer(fieldValue.UnsafeAddr())).Elem() + fieldValue.Set(deepCopy(fieldValue)) + } + return copy +} + +func deepCopyChan(original reflect.Value) reflect.Value { + return reflect.MakeChan(original.Type(), original.Cap()) +} + +// forceCopyValue simply creates a new pointer and sets its value to the original. +func forceCopyValue(original reflect.Value) reflect.Value { + originalType := original.Type() + newPointer := reflect.New(originalType) + newPointer.Elem().Set(original) + return newPointer.Elem() +} diff --git a/pkg/util/file.go b/pkg/util/file.go index 64ede022a9cc62ddf6b0796dd451d698c6ccc691..51b583d99247b66b01c11ef5ad4abea6ad0ec474 100644 --- a/pkg/util/file.go +++ b/pkg/util/file.go @@ -17,6 +17,7 @@ limitations under the License. package util import ( + "io/ioutil" "os" ) @@ -46,3 +47,27 @@ func WriteFile(filename string, data []byte, perm os.FileMode) error { } return nil } + +func ReadFile(filename string) ([]byte, error) { + file, err := os.Open(filename) + if err != nil { + return nil, err + } + defer file.Close() + contentByte, err := ioutil.ReadAll(file) + if err != nil { + return nil, err + } + return contentByte, nil +} + +func PathExists(path string) (bool, error) { + _, err := os.Stat(path) + if err == nil { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return false, err +} diff --git a/pkg/util/fs/file.go b/pkg/util/fs/file.go index d0edf1f5c18068d65ad6e9190036b5af7d9fcc84..ea6041b0949fc07bf18c11a5ac699d5da0fa4470 100644 --- a/pkg/util/fs/file.go +++ b/pkg/util/fs/file.go @@ -123,25 +123,37 @@ func Tar(src fs.FS, dst string) error { // Untar extracts the src tarball and saves to disk with path dst. func Untar(src, dst string) (err error) { - fr, err := os.Open(src) + var ( + fr *os.File + gr *gzip.Reader + hdr *tar.Header + file *os.File + ) + fr, err = os.Open(src) if err != nil { return err } defer func() { - err = fr.Close() + err1 := fr.Close() + if err == nil { + err = err1 + } }() - gr, err := gzip.NewReader(fr) + gr, err = gzip.NewReader(fr) if err != nil { return err } defer func() { - err = gr.Close() + err1 := gr.Close() + if err == nil { + err = err1 + } }() tr := tar.NewReader(gr) for { - hdr, err := tr.Next() + hdr, err = tr.Next() if err != nil { if err == io.EOF { return nil @@ -160,7 +172,7 @@ func Untar(src, dst string) (err error) { return err } case tar.TypeReg: - file, err := os.OpenFile(dirOrFile, os.O_CREATE|os.O_RDWR, os.FileMode(hdr.Mode)) + file, err = os.OpenFile(dirOrFile, os.O_CREATE|os.O_RDWR, os.FileMode(hdr.Mode)) if err != nil { return err } diff --git a/pkg/util/fs/fs.go b/pkg/util/fs/fs.go index a6b2bcd04ec877067f81924461a9c9aa24b1e24e..17be952a55844187eceb77e16c52629720d0b08b 100644 --- a/pkg/util/fs/fs.go +++ b/pkg/util/fs/fs.go @@ -90,3 +90,15 @@ func SaveFile(src io.ReadCloser, dst string) error { _, err = io.Copy(file, src) return err } + +// Determine whether the given path is a folder +func IsDir(path string) (bool, error) { + s, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + return s.IsDir(), nil +} diff --git a/pkg/util/helm.go b/pkg/util/helm.go index 6dfd76364b4a60b0a7f9f316c0967cdabea287b1..6cdf345a1eb310940c995d2e496f4ab915d18f16 100644 --- a/pkg/util/helm.go +++ b/pkg/util/helm.go @@ -17,14 +17,13 @@ limitations under the License. package util import ( - "fmt" "strings" ) -func GeneHelmReleaseName(namespace, serviceName string) string { - return fmt.Sprintf("%s-%s", namespace, serviceName) -} - -func ExtraServiceName(releaseName, namespace string) string { - return strings.TrimPrefix(releaseName, fmt.Sprintf("%s-", namespace)) +func GeneReleaseName(namingRule, projectName, namespace, envName, service string) string { + ret := strings.ReplaceAll(namingRule, "$Product$", projectName) + ret = strings.ReplaceAll(ret, "$Namespace$", namespace) + ret = strings.ReplaceAll(ret, "$EnvName$", envName) + ret = strings.ReplaceAll(ret, "$Service$", service) + return ret } diff --git a/pkg/util/pointer.go b/pkg/util/pointer.go new file mode 100644 index 0000000000000000000000000000000000000000..1c3ecd5711a365e0ba30eb54d2ce3d31b66037d7 --- /dev/null +++ b/pkg/util/pointer.go @@ -0,0 +1,29 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +func GetBoolPointer(data bool) *bool { + return &data +} + +func GetStrPointer(data string) *string { + return &data +} + +func GetInt32Pointer(data int32) *int32 { + return &data +} diff --git a/pkg/util/random.go b/pkg/util/random.go index 1cbd6b4dc289f9f331d039ed623093e7551b7e7b..0ae20360597aff672ce669fb553fe962da8dbed5 100644 --- a/pkg/util/random.go +++ b/pkg/util/random.go @@ -23,6 +23,10 @@ import ( "github.com/google/uuid" ) +func init() { + rand.Seed(time.Now().UnixNano()) +} + func UUID() string { return uuid.New().String() } diff --git a/pkg/util/strings.go b/pkg/util/strings.go index 91438a43c733035959890600df527f458b893914..6e3ec1126a3f71fb04d4095947ee3a3221e035ae 100644 --- a/pkg/util/strings.go +++ b/pkg/util/strings.go @@ -19,6 +19,8 @@ package util import ( "regexp" "strings" + + ref "github.com/containers/image/docker/reference" ) func GetJiraKeys(title string) (keys []string) { @@ -36,3 +38,26 @@ func ReplaceWrapLine(script string) string { -1, ), "\r", "\n", -1) } + +// Test case reference https://github.com/containers/image/blob/main/docker/reference/reference_test.go +func ExtractImageName(image string) string { + imageNameStr := "" + + reference, err := ref.Parse(image) + if err != nil { + return imageNameStr + } + if named, ok := reference.(ref.Named); ok { + imageNameArr := strings.Split(named.Name(), "/") + imageNameStr = imageNameArr[len(imageNameArr)-1] + } + + return imageNameStr +} + +func GetImageNameFromContainerInfo(imageName, containerName string) string { + if imageName == "" { + return containerName + } + return imageName +} diff --git a/pkg/util/url.go b/pkg/util/url.go index 8ca48093c24d2e52d475017ab1a788f46c99859a..8e054bfb0da93711ca2e14f5726224912fcba54b 100644 --- a/pkg/util/url.go +++ b/pkg/util/url.go @@ -20,8 +20,19 @@ import ( "fmt" "net/url" "strings" + + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types" ) +func TrimURLScheme(urlAddr string) string { + uri, err := url.Parse(strings.TrimSuffix(urlAddr, "/")) + if err != nil { + return urlAddr + } + return uri.Host + uri.Path +} + // GetURLHostName ... func GetURLHostName(urlAddr string) string { @@ -34,13 +45,28 @@ func GetURLHostName(urlAddr string) string { func ReplaceRepo(origin, addr, namespace string) string { parts := strings.SplitN(origin, "/", -1) + if namespace != "" { + return strings.Join([]string{ + TrimURLScheme(addr), + namespace, + parts[len(parts)-1], + }, "/") + } return strings.Join([]string{ - GetURLHostName(addr), - namespace, + TrimURLScheme(addr), parts[len(parts)-1], }, "/") } +func ReplaceTag(origin, newTag string) string { + parts := strings.Split(origin, ":") + if len(parts) != 2 { + return origin + } + parts[len(parts)-1] = newTag + return strings.Join(parts, ":") +} + func GetAddress(uri string) (string, error) { u, err := url.Parse(uri) if err != nil { @@ -52,3 +78,31 @@ func GetAddress(uri string) (string, error) { return fmt.Sprintf("%s://%s", u.Scheme, u.Host), nil } + +// ParseOwnerAndRepo extracts the owner and repo info from the given link, +// the link must have to following format: http(s)://example.com/owner/repo +func ParseOwnerAndRepo(repoLink string, authType types.AuthType) (string, string) { + var ownerAndRepo []string + repoLink = strings.TrimSuffix(repoLink, ".git") + switch authType { + case types.SSHAuthType: + repoLinkArr := strings.Split(repoLink, ":") + currentRepoLink := repoLinkArr[len(repoLinkArr)-1] + uriPath := strings.Trim(currentRepoLink, "/") + ownerAndRepo = strings.Split(uriPath, "/") + case types.PrivateAccessTokenAuthType: + uri, err := url.Parse(repoLink) + if err != nil { + log.Errorf("failed to parse url, err:%s", err) + return "", "" + } + + uriPath := strings.Trim(uri.Path, "/") + ownerAndRepo = strings.Split(uriPath, "/") + } + if len(ownerAndRepo) != 2 { + return "", "" + } + + return ownerAndRepo[0], ownerAndRepo[1] +} diff --git a/pkg/util/yaml.go b/pkg/util/yaml.go index 99c1f6848fa35bdfb9ac9826f1cdf4547ce42c85..330c3dd7a2e571142e9b0542bf16e06303f42e49 100644 --- a/pkg/util/yaml.go +++ b/pkg/util/yaml.go @@ -42,3 +42,14 @@ func SplitManifests(content string) []string { return res } + +// value may only contain underscores or underscores +func ReturnValidLabelValue(value string) string { + value = strings.Replace(value, "-", "", -1) + value = strings.Replace(value, "_", "", -1) + if len(value) > 63 { + value = value[:63] + } + + return value +} diff --git a/pkg/util/yaml/compare.go b/pkg/util/yaml/compare.go new file mode 100644 index 0000000000000000000000000000000000000000..230dc75cf930d7d106c370df17a6d4ca316e5e01 --- /dev/null +++ b/pkg/util/yaml/compare.go @@ -0,0 +1,43 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package yaml + +import ( + "reflect" + + "sigs.k8s.io/yaml" +) + +func Equal(source, target string) (bool, error) { + if source == target { + return true, nil + } + + sourceYamlObj, targetYamlObj := make(map[string]interface{}), make(map[string]interface{}) + + err := yaml.Unmarshal([]byte(source), &sourceYamlObj) + if err != nil { + return false, err + } + + err = yaml.Unmarshal([]byte(target), &targetYamlObj) + if err != nil { + return false, err + } + + return reflect.DeepEqual(sourceYamlObj, targetYamlObj), nil +} diff --git a/pkg/util/yaml/compare_test.go b/pkg/util/yaml/compare_test.go new file mode 100644 index 0000000000000000000000000000000000000000..8c624e40f031b69f2e4698b2a58cb57c0ed04f44 --- /dev/null +++ b/pkg/util/yaml/compare_test.go @@ -0,0 +1,75 @@ +/* +Copyright 2022 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package yaml + +import ( + . "github.com/onsi/ginkgo" + "github.com/onsi/gomega" +) + +var fromYaml = ` +env: dev +image: + repository: go-sample-site + tag: "0.2.1" +imagePullSecrets: + - name: default-secret +` + +var toYaml = ` +imagePullSecrets: + - name: default-secret +env: dev +image: + repository: go-sample-site + tag: "0.2.1" +` + +var toYaml2 = ` +# 这是一个注释 +# 这是一个注释2 +imagePullSecrets: + - name: default-secret +env: dev +image: + repository: go-sample-site + tag: "0.2.1" +` + +var _ = Describe("Testing yaml equal", func() { + + Context("yaml equal", func() { + It("should work as expected", func() { + equal, err := Equal(fromYaml, toYaml) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(equal).Should(gomega.BeTrue()) + }) + + It("should work as expected", func() { + equal, err := Equal(fromYaml, toYaml2) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(equal).Should(gomega.BeTrue()) + }) + + It("should work as expected", func() { + equal, err := Equal(toYaml, toYaml2) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(equal).Should(gomega.BeTrue()) + }) + + }) +}) diff --git a/pkg/util/yaml/path_search.go b/pkg/util/yaml/path_search.go index 8f09c6b08517e071aa9fda0f3534e0140468e411..919475cfffeaa4f894fc078bf785cff6e8c69575 100644 --- a/pkg/util/yaml/path_search.go +++ b/pkg/util/yaml/path_search.go @@ -19,7 +19,14 @@ package yaml import ( "errors" "fmt" + "math" "strings" + + "k8s.io/utils/strings/slices" +) + +const ( + separator = "." ) func getKey(prefix, k string) string { @@ -59,7 +66,227 @@ type pathSearcher struct { resultSet map[string]*pathSearchRuntime //prefix => search result } -func (isr *pathSearcher) handleKV(k string, v interface{}) { +type patternDataMap map[string]map[string]*patternData + +// check if contains empty patterns +func preFilterSearchPaths(configs []map[string]string) error { + for _, singleConfig := range configs { + for k, v := range singleConfig { + if k == "" || v == "" { + return errors.New("empty pattern name or pattern path is not supported") + } + } + } + return nil +} + +func (pdm patternDataMap) addPatternMap(name, rPath, aPath string) *patternData { + pData := &patternData{ + name: name, + relativePath: rPath, + absolutePath: aPath, + } + if _, ok := pdm[name]; !ok { + pdm[name] = make(map[string]*patternData) + } + pdm[name][aPath] = pData + return pData +} + +func (pdm patternDataMap) getAPathByName(name string) []string { + ret := make([]string, 0) + if pData, ok := pdm[name]; ok { + for _, data := range pData { + ret = append(ret, data.absolutePath) + } + } + return ret +} + +type patternData struct { + name string + relativePath string + absolutePath string + usedPrefix string +} + +type pathResult struct { + name string + absolutePath string +} + +type pathResultSet struct { + resultSet []*pathResult + prefix string +} + +// paths share the same longest prefix +type relativePathSet struct { + paths []string +} + +type relativePathChecker struct { + relativePaths map[string]*relativePathSet +} + +func strSliceContainsAnother(source, target []string) bool { + for _, str := range target { + if !slices.Contains(source, str) { + return false + } + } + return true +} + +// check if the input path set contains relative paths +func (checker *relativePathChecker) checkMatchRelativePath(paths []string) bool { + for _, path := range paths { + if relativePath, ok := checker.relativePaths[path]; ok { + if strSliceContainsAnother(paths, relativePath.paths) { + return true + } else { + return false + } + } + } + return true +} + +func newRelativeChecker(rPaths [][]string) *relativePathChecker { + ret := &relativePathChecker{relativePaths: map[string]*relativePathSet{}} + for _, pathSet := range rPaths { + relativeData := &relativePathSet{ + paths: pathSet, + } + for _, path := range pathSet { + ret.relativePaths[path] = relativeData + } + } + return ret +} + +// find the longest common prefix from string list +func longestCommonPrefix(strList []string) string { + n := len(strList) + if n == 0 { + return "" + } else if n == 1 { + return strList[0] + } + minStrLen := math.MaxInt32 + for i := 0; i < n; i++ { + if minStrLen > len(strList[i]) { + minStrLen = len(strList[i]) + } + } + prefix := strList[0][0:minStrLen] + var allFound bool + for { + allFound = true + for i := 1; i < n; i++ { + if strings.Index(strList[i], prefix) != 0 { + prefix = prefix[0 : len(prefix)-1] + allFound = false + break + } + } + if allFound || len(prefix) == 0 { + break + } + } + return prefix +} + +func findCombinations(arr [][]string, checker *relativePathChecker) [][]string { + ret := make([][]string, 0) + n := len(arr) + indices := make([]int, n, n) + for { + singleSlice := make([]string, 0) + for i := 0; i < n; i++ { + singleSlice = append(singleSlice, arr[i][indices[i]]) + } + if len(singleSlice) > 0 { + if checker.checkMatchRelativePath(singleSlice) { + ret = append(ret, singleSlice) + } + } + + // find the rightmost array that has more + // elements left after the current element + // in that array + next := n - 1 + for { + if next >= 0 && (indices[next]+1 >= len(arr[next])) { + next-- + } else { + break + } + } + if next < 0 { + break + } + // if found move to next element in that + // array + indices[next] = indices[next] + 1 + + // for all arrays to the right of this + // array current index again points to + // first element + for i := next + 1; i < n; i++ { + indices[i] = 0 + } + } + + return ret +} + +func toPathResult(prefix string, path []string, aPatternData map[string]*patternData) *pathResultSet { + ret := &pathResultSet{ + prefix: prefix, + } + for _, aPath := range path { + pData, _ := aPatternData[aPath] + if len(pData.usedPrefix) <= len(prefix) { + pData.usedPrefix = prefix + } + ret.resultSet = append(ret.resultSet, &pathResult{ + name: pData.name, + absolutePath: aPath, + }) + } + return ret +} + +func processResults(resultSets []*pathResultSet, aPatternData map[string]*patternData, pattern map[string]string) []map[string]string { + ret := make([]map[string]string, 0) + for _, singleSet := range resultSets { + retMap := make(map[string]string) + + useLongestPrefix := true + for _, result := range singleSet.resultSet { + pData := aPatternData[result.absolutePath] + if len(pData.usedPrefix) != len(singleSet.prefix) { + useLongestPrefix = false + break + } + } + if !useLongestPrefix { + continue + } + + for _, result := range singleSet.resultSet { + retMap[result.name] = result.absolutePath + } + if len(pattern) != len(retMap) { + continue + } + ret = append(ret, retMap) + } + return ret +} + +func (isr *pathSearcher) checkRelativePath(k string) { expectPaths, preMatch := isr.pattern, false for _, ep := range expectPaths { if strings.HasSuffix(k, ep) { @@ -95,7 +322,6 @@ func (isr *pathSearcher) handleKV(k string, v interface{}) { pathName: pathName, relativePath: ep, absolutePath: k, - value: v, } match = true } @@ -119,7 +345,6 @@ func (isr *pathSearcher) handleKV(k string, v interface{}) { pathName: pathName, relativePath: ep, absolutePath: k, - value: v, }, }, } @@ -128,69 +353,65 @@ func (isr *pathSearcher) handleKV(k string, v interface{}) { } } -// check if contains empty patterns -func preFilterSearchPaths(configs []map[string]string) error { - for _, singleConfig := range configs { - for k, v := range singleConfig { - if k == "" || v == "" { - return errors.New("empty pattern name or pattern path is not supported") - } +func (isr *pathSearcher) getRelativePath() [][]string { + ret := make([][]string, 0) + for _, fSet := range isr.resultSet { + // multi keys share the same prefix + if len(fSet.foundSet) <= 1 { + continue } + set := make([]string, 0) + for _, searchInfo := range fSet.foundSet { + set = append(set, searchInfo.absolutePath) + } + ret = append(ret, set) } - return nil + return ret } -// build pathSearcher object for every pattern -func searchPaths(patterns []map[string]string, sourceMap map[string]interface{}) ([]*pathSearcher, error) { - if len(patterns) == 0 { - return nil, nil - } - err := preFilterSearchPaths(patterns) - if err != nil { - return nil, err +func search(flatMap map[string]interface{}, pattern map[string]string) ([]map[string]string, error) { + + patternDataMap := patternDataMap{} + allPatternData := make(map[string]*patternData) // absolutePath => patternData + relativeSearcher := &pathSearcher{ + pattern: pattern, + resultSet: make(map[string]*pathSearchRuntime), } - rtSlice := make([]*pathSearcher, 0) - for _, cfg := range patterns { - rt := &pathSearcher{ - pattern: cfg, - resultSet: make(map[string]*pathSearchRuntime), + for absolutePath := range flatMap { + for patternName, relativePath := range pattern { + if relativePath != absolutePath && !strings.HasSuffix(absolutePath, fmt.Sprintf(".%s", relativePath)) { + continue + } + pData := patternDataMap.addPatternMap(patternName, relativePath, absolutePath) + allPatternData[absolutePath] = pData } - rtSlice = append(rtSlice, rt) + relativeSearcher.checkRelativePath(absolutePath) } - for k, v := range sourceMap { - for _, rt := range rtSlice { - rt.handleKV(k, v) + + relativePaths := relativeSearcher.getRelativePath() + checker := newRelativeChecker(relativePaths) + + allKeys := make([][]string, 0) + for name, _ := range patternDataMap { + pathList := patternDataMap.getAPathByName(name) + if len(pathList) == 0 { + return nil, nil } + allKeys = append(allKeys, pathList) } - return rtSlice, nil -} -// merge results and filter duplicates paths -func mergeResults(isrList []*pathSearcher) []map[string]string { - ret := make([]map[string]string, 0) - usedPath := make(map[string]int) - for _, isr := range isrList { - for _, fSet := range isr.resultSet { - retSet := make(map[string]string) - if !fSet.checkAllFinish() { - continue - } - duplicateCheckPass := true - for _, searchInfo := range fSet.foundSet { - if _, ok := usedPath[searchInfo.absolutePath]; ok { - duplicateCheckPass = false - break - } - retSet[searchInfo.pathName] = searchInfo.absolutePath - usedPath[searchInfo.absolutePath] = 1 - } - if duplicateCheckPass { - ret = append(ret, retSet) - } - } + allCombinations := findCombinations(allKeys, checker) + + results := make([]*pathResultSet, 0) + + for _, cb := range allCombinations { + lp := longestCommonPrefix(cb) + result := toPathResult(lp, cb, allPatternData) + results = append(results, result) } - return ret + + return processResults(results, allPatternData, pattern), nil } // SearchByPattern find all matched absolute paths from yaml by the pattern appointed @@ -212,11 +433,19 @@ func mergeResults(isrList []*pathSearcher) []map[string]string { // ----------- yaml end // pattern: []{"image": "repository", "tag": "tag"} // output: []{{"image": "svc1.image.repository", "tag": "svc1.image.tag"}, {"image": "svc2.image.repository", "tag": "svc2.image.tag"}} -func SearchByPattern(flatMap map[string]interface{}, pattern []map[string]string) ([]map[string]string, error) { - foundResult, err := searchPaths(pattern, flatMap) +func SearchByPattern(flatMap map[string]interface{}, patterns []map[string]string) ([]map[string]string, error) { + err := preFilterSearchPaths(patterns) if err != nil { return nil, err } - ret := mergeResults(foundResult) + + ret := make([]map[string]string, 0) + for _, singlePattern := range patterns { + pRet, err := search(flatMap, singlePattern) + if err != nil { + return nil, err + } + ret = append(ret, pRet...) + } return ret, nil } diff --git a/pkg/util/yaml/path_search_test.go b/pkg/util/yaml/path_search_test.go index bdf48b385e9410480863ea3b915c37fbd3294a09..bf285e6bf12fa36cf4ddad44c4db2c45bef6c6d4 100644 --- a/pkg/util/yaml/path_search_test.go +++ b/pkg/util/yaml/path_search_test.go @@ -18,7 +18,7 @@ package yaml import ( . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" + "github.com/onsi/gomega" "github.com/koderover/zadig/pkg/util/converter" ) @@ -26,7 +26,8 @@ import ( var testYaml1 = ` env: dev image: - repository: go-sample-site + repo: repo.com + name: go-sample-site tag: "0.2.1" imagePullSecrets: - name: default-secret @@ -36,12 +37,14 @@ var testYaml2 = ` env: dev svc1: image: - repository: go-sample-site + repo: repo.com + name: go-sample-site tag: "0.2.1" svc2: image: - repository: go-sample-site-2 - tag: "0.2.2" + repo: repo.com + name: go-sample-site + tag: "0.2.1" imagePullSecrets: - name: default-secret ` @@ -50,61 +53,42 @@ var testYaml3 = ` env: dev svc1: image: - repository: go-sample-site:0.2.1 + repo: repo.com + name: go-sample-site + tag: "0.2.1" svc2: image: - repository: go-sample-site-2:0.2.2 -svc3: - image: - repository: go-sample-site-3:0.2.3 + repo: repo.com + detail: + name: go-sample-site + tag: "0.2.1" +fakeSvc: + repo: notrepo.com imagePullSecrets: - name: default-secret ` -var testYaml4 = ` +var testYaml8 = ` env: dev -svc1: - image: - repository: go-sample-site - tag: 0.2.1 -svc2: - image: - repository: go-sample-site-2:0.2.2 -svc3: - image: - repository: go-sample-site-3:0.2.3 -svc4: - image: - repositoryNew: go-sample-site-3 - tagNew: 0.2.4 -svc5: - second: - image: - repositorySpec: go-sample-site-3 - tagNew: 0.2.4 -imagePullSecrets: - - name: default-secret -` - -var testYaml5 = ` -# Default values for go-sample-site. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -env: dev - -ingressClassName: koderover-admin-nginx -image: - repository: ccr.ccs.tencentyun.com/trial/go-sample-site - pullPolicy: IfNotPresent - tag: "0.1.0" +repoData1: + global: + hub: koderover.tencentcloudcr.com/koderover-public testSpec: - imageNew: - repo: ccr.ccs.tencentyun.com/trial/go-sample-site-new + image: + image: go-sample-site-new pullPolicy: IfNotPresent - tag: "0.2.1" + tag: 0.1.0 + +svc1: + image: svc1-image + tag: 0.2.0 + +svc2: + image: + image: svc2-image + tag: 0.3.0 imagePullSecrets: - name: default-registry-secret @@ -115,81 +99,114 @@ fullnameOverride: "" service: type: ClusterIP port: 8080 +` + +var testYaml9 = ` + image: + pullPolicy: IfNotPresent + + imagePullSecrets: + - name: default-registry-secret + global: + hub: koderover.tencentcloudcr.com/test + + deploy: + image: + name: go-sample-site + tag: 0.1.0 ` var err error -var matedPaths []map[string]string +var lcpMatedPaths []map[string]string var _ = Describe("Testing search", func() { Context("search matched paths from yaml", func() { - It("single match", func() { + It("single absolute match", func() { pattern := []map[string]string{ - {"image": "repository", "tag": "tag"}, + {"repo": "repo", "tag": "tag", "image": "name"}, } flatMap, _ := converter.YamlToFlatMap([]byte(testYaml1)) - matedPaths, err = SearchByPattern(flatMap, pattern) - Expect(err).NotTo(HaveOccurred()) - Expect(matedPaths).To(Equal([]map[string]string{{"image": "image.repository", "tag": "image.tag"}})) + lcpMatedPaths, err = SearchByPattern(flatMap, pattern) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(lcpMatedPaths).Should(gomega.ConsistOf([]map[string]string{ + {"image": "image.name", "repo": "image.repo", "tag": "image.tag"}, + })) + }) + + It("single relative match", func() { + pattern := []map[string]string{ + {"repo": "image.repo", "tag": "image.tag", "image": "image.name"}, + } + flatMap, _ := converter.YamlToFlatMap([]byte(testYaml1)) + lcpMatedPaths, err = SearchByPattern(flatMap, pattern) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(lcpMatedPaths).Should(gomega.ConsistOf([]map[string]string{ + {"image": "image.name", "repo": "image.repo", "tag": "image.tag"}, + })) }) It("multiple match", func() { pattern := []map[string]string{ - {"image": "repository", "tag": "tag"}, + {"repo": "repo", "tag": "tag", "image": "name"}, } flatMap, _ := converter.YamlToFlatMap([]byte(testYaml2)) - matedPaths, err = SearchByPattern(flatMap, pattern) - Expect(err).NotTo(HaveOccurred()) - Expect(matedPaths).Should(ConsistOf([]map[string]string{ - {"image": "svc1.image.repository", "tag": "svc1.image.tag"}, - {"image": "svc2.image.repository", "tag": "svc2.image.tag"}, + lcpMatedPaths, err = SearchByPattern(flatMap, pattern) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + gomega.Expect(lcpMatedPaths).Should(gomega.ConsistOf([]map[string]string{ + {"image": "svc1.image.name", "repo": "svc1.image.repo", "tag": "svc1.image.tag"}, + {"image": "svc2.image.name", "repo": "svc2.image.repo", "tag": "svc2.image.tag"}, })) }) - It("multiple match pattern 3", func() { + It("complex multiple match", func() { pattern := []map[string]string{ - {"image": "repository"}, + {"repo": "repo", "tag": "tag", "image": "name"}, } - flatMap, _ := converter.YamlToFlatMap([]byte(testYaml3)) - matedPaths, err = SearchByPattern(flatMap, pattern) - Expect(err).NotTo(HaveOccurred()) - Expect(matedPaths).Should(ConsistOf([]map[string]string{ - {"image": "svc1.image.repository"}, - {"image": "svc2.image.repository"}, - {"image": "svc3.image.repository"}, + flatMap, err := converter.YamlToFlatMap([]byte(testYaml3)) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + lcpMatedPaths, err = SearchByPattern(flatMap, pattern) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + gomega.Expect(lcpMatedPaths).Should(gomega.ConsistOf([]map[string]string{ + {"image": "svc1.image.name", "repo": "svc1.image.repo", "tag": "svc1.image.tag"}, + {"image": "svc2.image.detail.name", "repo": "svc2.image.repo", "tag": "svc2.image.detail.tag"}, })) }) - It("multiple match pattern complex", func() { + It("complex match with pattern component reuse", func() { pattern := []map[string]string{ - {"image": "repository", "tag": "tag"}, - {"image": "repository"}, - {"image": "repositoryNew", "tag": "tagNew"}, - {"image": "image.repositorySpec", "tag": "tagNew"}, + {"image": "image", "tag": "tag", "repo": "global.hub"}, } - flatMap, _ := converter.YamlToFlatMap([]byte(testYaml4)) - matedPaths, err = SearchByPattern(flatMap, pattern) - Expect(err).NotTo(HaveOccurred()) - Expect(matedPaths).Should(ConsistOf([]map[string]string{ - {"image": "svc1.image.repository", "tag": "svc1.image.tag"}, - {"image": "svc2.image.repository"}, - {"image": "svc3.image.repository"}, - {"image": "svc4.image.repositoryNew", "tag": "svc4.image.tagNew"}, - {"image": "svc5.second.image.repositorySpec", "tag": "svc5.second.tagNew"}, - })) + flatMap, err := converter.YamlToFlatMap([]byte(testYaml8)) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + lcpMatedPaths, err = SearchByPattern(flatMap, pattern) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(lcpMatedPaths).Should(gomega.ConsistOf([]map[string]string{ + {"image": "testSpec.image.image", "repo": "repoData1.global.hub", "tag": "testSpec.image.tag"}, + {"image": "svc1.image", "repo": "repoData1.global.hub", "tag": "svc1.tag"}, + {"image": "svc2.image.image", "repo": "repoData1.global.hub", "tag": "svc2.image.tag"}, + })) }) - It("multiple match pattern complex2", func() { + It("match all rules of pattern", func() { pattern := []map[string]string{ - {"image": "imageNew.repo", "tag": "imageNew.tag"}, + {"image": "image.repository", "tag": "image.tag"}, + {"image": "image"}, + {"repo": "global.hub", "tag": "deploy.image.tag", "image": "deploy.image.name"}, } - flatMap, _ := converter.YamlToFlatMap([]byte(testYaml5)) - matedPaths, err = SearchByPattern(flatMap, pattern) - Expect(err).NotTo(HaveOccurred()) - Expect(len(matedPaths)).To(Equal(1)) - Expect(matedPaths).To(Equal([]map[string]string{{"image": "testSpec.imageNew.repo", "tag": "testSpec.imageNew.tag"}})) + flatMap, _ := converter.YamlToFlatMap([]byte(testYaml9)) + lcpMatedPaths, err = SearchByPattern(flatMap, pattern) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(lcpMatedPaths).Should(gomega.ConsistOf([]map[string]string{ + {"image": "deploy.image.name", "repo": "global.hub", "tag": "deploy.image.tag"}, + })) }) + }) }) diff --git a/ut.file b/ut.file index 046285034f167f82b0c62d2d001ea3936aab9e3c..338593ddf1b74e60e004bfd5a370ffb0ef4df6e3 100644 --- a/ut.file +++ b/ut.file @@ -3,4 +3,4 @@ ./pkg/microservice/reaper/core/service/meta ./pkg/microservice/reaper/core/service/reaper ./pkg/microservice/aslan/core/common/service/registry -./pkg/util/converter +./pkg/util/converter \ No newline at end of file diff --git a/version/version.go b/version/version.go index c95391039ed4eee585a2ec94510725980df07c34..550858137e42128a7d8ba6dc06ca54b8c673b2ce 100644 --- a/version/version.go +++ b/version/version.go @@ -17,7 +17,7 @@ limitations under the License. package version var ( - Version = "1.5.0" + Version = "1.9.0" BuildNumber = "0" GitCommit = "" )