37 Star 396 Fork 71

GVPrancher / rancher

加入 Gitee
与超过 1200万 开发者一起发现、参与优秀开源项目,私有仓库也完全免费 :)
免费加入
克隆/下载
engine.go 21.13 KB
一键复制 编辑 原始数据 按行查看 历史
phli 提交于 2018-10-18 18:17 . secure string conversion for registry fields
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664
package jenkins
import (
"encoding/xml"
"fmt"
"bytes"
"encoding/json"
"github.com/pkg/errors"
"github.com/rancher/norman/httperror"
"github.com/rancher/norman/types/convert"
"github.com/rancher/rancher/pkg/pipeline/utils"
"github.com/rancher/rancher/pkg/ref"
"github.com/rancher/types/apis/apps/v1beta2"
"github.com/rancher/types/apis/core/v1"
"github.com/rancher/types/apis/project.cattle.io/v3"
"github.com/rancher/types/config/dialer"
"github.com/sirupsen/logrus"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"net/http"
"regexp"
"strconv"
"strings"
"time"
)
type Engine struct {
JenkinsClient *Client
HTTPClient *http.Client
ServiceLister v1.ServiceLister
PodLister v1.PodLister
DeploymentLister v1beta2.DeploymentLister
Secrets v1.SecretInterface
SecretLister v1.SecretLister
ManagementSecretLister v1.SecretLister
SourceCodeCredentialLister v3.SourceCodeCredentialLister
PipelineLister v3.PipelineLister
ClusterName string
Dialer dialer.Factory
}
func (j *Engine) getJenkinsURL(execution *v3.PipelineExecution) (string, error) {
ns := utils.GetPipelineCommonName(execution)
service, err := j.ServiceLister.Get(ns, utils.JenkinsName)
if err != nil {
return "", err
}
ip := service.Spec.ClusterIP
return fmt.Sprintf("http://%s:%d", ip, utils.JenkinsPort), nil
}
func (j *Engine) PreCheck(execution *v3.PipelineExecution) (bool, error) {
var pod *corev1.Pod
var err error
set := labels.Set(map[string]string{utils.LabelKeyApp: utils.JenkinsName})
ns := utils.GetPipelineCommonName(execution)
pods, err := j.PodLister.List(ns, set.AsSelector())
if err != nil {
return false, err
}
if len(pods) <= 0 {
return false, errors.New("jenkins pod not found")
}
pod = pods[0]
for _, cond := range pod.Status.Conditions {
if cond.Type == corev1.PodReady && cond.Status == corev1.ConditionTrue {
return true, nil
}
}
return false, nil
}
func (j *Engine) getJenkinsClient(execution *v3.PipelineExecution) (*Client, error) {
url, err := j.getJenkinsURL(execution)
if err != nil {
return nil, err
}
user := utils.PipelineSecretDefaultUser
ns := utils.GetPipelineCommonName(execution)
secret, err := j.SecretLister.Get(ns, utils.PipelineSecretName)
if err != nil || secret.Data == nil {
return nil, fmt.Errorf("error get jenkins token - %v", err)
}
token := string(secret.Data[utils.PipelineSecretTokenKey])
if j.HTTPClient == nil {
dial, err := j.Dialer.ClusterDialer(j.ClusterName)
if err != nil {
return nil, err
}
j.HTTPClient = &http.Client{
Transport: &http.Transport{
Dial: dial,
},
Timeout: 15 * time.Second,
}
}
return New(url, user, token, j.HTTPClient)
}
func (j *Engine) RunPipelineExecution(execution *v3.PipelineExecution) error {
logrus.Debug("start RunPipelineExecution")
jobName := getJobName(execution)
client, err := j.getJenkinsClient(execution)
if err != nil {
return err
}
_, err = client.getJobInfo(jobName)
if e, ok := err.(*httperror.APIError); ok && e.Code.Status == http.StatusNotFound {
if err = j.createPipelineJob(client, execution); err != nil {
return err
}
} else if err != nil {
return err
}
if err := j.updatePipelineJob(client, execution); err != nil {
return err
}
ns, name := ref.Parse(execution.Spec.PipelineName)
pipeline, err := j.PipelineLister.Get(ns, name)
if err != nil {
return err
}
if err := j.setCredential(client, execution, pipeline.Spec.SourceCodeCredentialName); err != nil {
return err
}
if err := j.preparePipeline(execution); err != nil {
return err
}
if _, err := client.buildJob(jobName, map[string]string{}); err != nil {
return err
}
return nil
}
func (j *Engine) preparePipeline(execution *v3.PipelineExecution) error {
for _, stage := range execution.Spec.PipelineConfig.Stages {
for _, step := range stage.Steps {
if step.PublishImageConfig != nil {
//prepare docker credential for publishimage step
registry := utils.DefaultRegistry
if step.PublishImageConfig.PushRemote && step.PublishImageConfig.Registry != "" {
registry = step.PublishImageConfig.Registry
} else {
_, projectID := ref.Parse(execution.Spec.ProjectName)
registry = fmt.Sprintf("%s.%s-pipeline", utils.LocalRegistry, projectID)
}
if err := j.prepareRegistryCredential(execution, registry); err != nil {
return err
}
}
}
}
return nil
}
func (j *Engine) prepareRegistryCredential(execution *v3.PipelineExecution, registry string) error {
secrets, err := j.ManagementSecretLister.List(execution.Namespace, labels.Everything())
if err != nil {
return err
}
username := ""
password := ""
for _, s := range secrets {
if s.Type == "kubernetes.io/dockerconfigjson" {
m := map[string]interface{}{}
if err := json.Unmarshal(s.Data[".dockerconfigjson"], &m); err != nil {
return err
}
auths := convert.ToMapInterface(m["auths"])
for k, v := range auths {
if registry != k {
//find matching registry credential
continue
}
cred := convert.ToMapInterface(v)
username, _ = cred["username"].(string)
password, _ = cred["password"].(string)
}
}
}
//store dockercredential in pipeline namespace
//TODO key-key mapping instead of registry-key mapping
reg, _ := regexp.Compile("[^a-zA-Z0-9]+")
proceccedRegistry := strings.ToLower(reg.ReplaceAllString(registry, ""))
secretName := fmt.Sprintf("%s-%s", execution.Namespace, proceccedRegistry)
logrus.Debugf("preparing registry credential %s for %s", secretName, registry)
ns := utils.GetPipelineCommonName(execution)
secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Namespace: ns,
Name: secretName,
},
Data: map[string][]byte{
utils.PublishSecretUserKey: []byte(username),
utils.PublishSecretPwKey: []byte(password),
},
}
_, err = j.Secrets.Create(secret)
if apierrors.IsAlreadyExists(err) {
if _, err := j.Secrets.Update(secret); err != nil {
return err
}
return nil
}
return err
}
func (j *Engine) createPipelineJob(client *Client, execution *v3.PipelineExecution) error {
logrus.Debug("create jenkins job for pipeline")
jobconf, err := ConvertPipelineExecutionToJenkinsPipeline(execution)
if err != nil {
return err
}
jobName := getJobName(execution)
bconf, _ := xml.MarshalIndent(jobconf, " ", " ")
return client.createJob(jobName, bconf)
}
func (j *Engine) updatePipelineJob(client *Client, execution *v3.PipelineExecution) error {
logrus.Debug("update jenkins job for pipeline")
jobconf, err := ConvertPipelineExecutionToJenkinsPipeline(execution)
if err != nil {
return err
}
jobName := getJobName(execution)
bconf, _ := xml.MarshalIndent(jobconf, " ", " ")
return client.updateJob(jobName, bconf)
}
func (j *Engine) RerunExecution(execution *v3.PipelineExecution) error {
return j.RunPipelineExecution(execution)
}
func (j *Engine) StopExecution(execution *v3.PipelineExecution) error {
jobName := getJobName(execution)
client, err := j.getJenkinsClient(execution)
if err != nil {
return err
}
info, err := client.getJobInfo(jobName)
if e, ok := err.(*httperror.APIError); ok && e.Code.Status == http.StatusNotFound {
return nil
} else if err != nil {
return err
}
if info.InQueue {
//delete in queue
queueItem, ok := info.QueueItem.(map[string]interface{})
if !ok {
return fmt.Errorf("type assertion fail for queueitem")
}
queueID, ok := queueItem["id"].(float64)
if !ok {
return fmt.Errorf("type assertion fail for queueID")
}
if err := client.cancelQueueItem(int(queueID)); err != nil {
return fmt.Errorf("cancel queueitem error:%v", err)
}
} else {
buildInfo, err := client.getBuildInfo(jobName)
if err != nil {
return err
}
if buildInfo.Building {
if err := client.stopJob(jobName, buildInfo.Number); err != nil {
return err
}
}
}
return nil
}
func (j *Engine) SyncExecution(execution *v3.PipelineExecution) (bool, error) {
updated := false
jobName := getJobName(execution)
client, err := j.getJenkinsClient(execution)
if err != nil {
return false, err
}
buildinfo, err := client.getBuildInfo(jobName)
if e, ok := err.(*httperror.APIError); ok && e.Code.Status == http.StatusNotFound {
return false, nil
} else if err != nil {
return false, err
}
if buildinfo.Result == "FAILURE" {
//some errors are disclosed in buildinfo but not in wfbuildinfo
execution.Status.ExecutionState = utils.StateFailed
v3.PipelineExecutionConditionBuilt.False(execution)
v3.PipelineExecutionConditionBuilt.Message(execution, "buildinfo result failure")
updated = true
}
info, err := client.getWFBuildInfo(jobName)
if e, ok := err.(*httperror.APIError); ok && e.Code.Status == http.StatusNotFound {
return false, nil
} else if err != nil {
return false, err
}
for _, jenkinsStage := range info.Stages {
//handle those in step-1-1 format
parts := strings.Split(jenkinsStage.Name, "-")
if len(parts) == 3 {
stage, err := strconv.Atoi(parts[1])
if err != nil {
return false, err
}
step, err := strconv.Atoi(parts[2])
if err != nil {
return false, err
}
if len(execution.Status.Stages) <= stage || len(execution.Status.Stages[stage].Steps) <= step {
return false, errors.New("error sync execution - index out of range")
}
status := jenkinsStage.Status
if status == "SUCCESS" && execution.Status.Stages[stage].Steps[step].State != utils.StateSuccess {
if jenkinsStage.DurationMillis < 100 {
//For concurrent steps, there are cases Jenkins wfapi returns SUCCESS status while it it still IN_PROGRESS
updated = true
buildingStep(execution, stage, step, jenkinsStage)
continue
}
updated = true
if err := j.successStep(execution, stage, step, jenkinsStage); err != nil {
return false, err
}
} else if status == "FAILED" && execution.Status.Stages[stage].Steps[step].State != utils.StateFailed {
updated = true
if err := j.failStep(execution, stage, step, jenkinsStage); err != nil {
return false, err
}
} else if status == "IN_PROGRESS" && execution.Status.Stages[stage].Steps[step].State != utils.StateBuilding {
updated = true
buildingStep(execution, stage, step, jenkinsStage)
} else if status == "NOT_EXECUTED" && execution.Status.Stages[stage].Steps[step].State != utils.StateSkipped {
updated = true
skipStep(execution, stage, step, jenkinsStage)
}
}
}
if info.Status == "SUCCESS" && execution.Status.ExecutionState != utils.StateSuccess {
updated = true
execution.Labels[utils.PipelineFinishLabel] = "true"
execution.Status.ExecutionState = utils.StateSuccess
v3.PipelineExecutionConditionProvisioned.True(execution)
v3.PipelineExecutionConditionBuilt.True(execution)
} else if info.Status == "FAILED" && execution.Status.ExecutionState != utils.StateAborted &&
execution.Status.ExecutionState != utils.StateFailed {
updated = true
execution.Labels[utils.PipelineFinishLabel] = "true"
execution.Status.ExecutionState = utils.StateFailed
if v3.PipelineExecutionConditionProvisioned.IsUnknown(execution) {
v3.PipelineExecutionConditionProvisioned.True(execution)
}
v3.PipelineExecutionConditionBuilt.False(execution)
v3.PipelineExecutionConditionBuilt.Message(execution, "Buildinfo got FAILED status")
} else if info.Status == "IN_PROGRESS" && execution.Status.ExecutionState == utils.StateWaiting {
updated = true
execution.Status.ExecutionState = utils.StateBuilding
}
if execution.Status.ExecutionState == utils.StateBuilding {
if len(execution.Status.Stages) > 0 &&
len(execution.Status.Stages[0].Steps) > 0 &&
execution.Status.Stages[0].Steps[0].State == utils.StateWaiting {
//update ProvisionCondition
prepareLog, err := client.getWFNodeLog(jobName, PrepareWFNodeID)
if e, ok := err.(*httperror.APIError); ok && e.Code.Status == http.StatusNotFound {
return false, nil
} else if err != nil {
return false, err
}
prevMessage := v3.PipelineExecutionConditionProvisioned.GetMessage(execution)
curMessage := translatePreparingMessage(prepareLog.Text)
if prevMessage != curMessage && curMessage != "" {
v3.PipelineExecutionConditionProvisioned.Message(execution, curMessage)
updated = true
}
} else {
v3.PipelineExecutionConditionProvisioned.True(execution)
}
}
return updated, nil
}
func (j *Engine) successStep(execution *v3.PipelineExecution, stage int, step int, jenkinsStage Stage) error {
startTime := time.Unix(jenkinsStage.StartTimeMillis/1000, 0).Format(time.RFC3339)
endTime := time.Unix((jenkinsStage.StartTimeMillis+jenkinsStage.DurationMillis)/1000, 0).Format(time.RFC3339)
execution.Status.Stages[stage].Steps[step].State = utils.StateSuccess
if execution.Status.Stages[stage].Steps[step].Started == "" {
execution.Status.Stages[stage].Steps[step].Started = startTime
}
execution.Status.Stages[stage].Steps[step].Ended = endTime
if execution.Status.Stages[stage].Started == "" {
execution.Status.Stages[stage].Started = startTime
}
if execution.Status.Started == "" {
execution.Status.Started = startTime
}
if utils.IsStageSuccess(execution.Status.Stages[stage]) {
execution.Status.Stages[stage].State = utils.StateSuccess
execution.Status.Stages[stage].Ended = endTime
if stage == len(execution.Status.Stages)-1 {
execution.Status.ExecutionState = utils.StateSuccess
execution.Status.Ended = endTime
execution.Labels[utils.PipelineFinishLabel] = "true"
v3.PipelineExecutionConditionBuilt.True(execution)
}
}
if err := j.saveStepLogToMinio(execution, stage, step); err != nil {
return err
}
return nil
}
func (j *Engine) failStep(execution *v3.PipelineExecution, stage int, step int, jenkinsStage Stage) error {
startTime := time.Unix(jenkinsStage.StartTimeMillis/1000, 0).Format(time.RFC3339)
endTime := time.Unix((jenkinsStage.StartTimeMillis+jenkinsStage.DurationMillis)/1000, 0).Format(time.RFC3339)
execution.Status.Stages[stage].Steps[step].State = utils.StateFailed
execution.Status.Stages[stage].State = utils.StateFailed
if execution.Status.ExecutionState != utils.StateAborted {
execution.Status.ExecutionState = utils.StateFailed
v3.PipelineExecutionConditionBuilt.False(execution)
v3.PipelineExecutionConditionBuilt.Message(execution, fmt.Sprintf("Got FAILED status in '%s' stage", execution.Spec.PipelineConfig.Stages[stage].Name))
}
if execution.Status.Stages[stage].Steps[step].Started == "" {
execution.Status.Stages[stage].Steps[step].Started = startTime
}
execution.Status.Stages[stage].Steps[step].Ended = endTime
if execution.Status.Stages[stage].Started == "" {
execution.Status.Stages[stage].Started = startTime
}
if execution.Status.Stages[stage].Ended == "" {
execution.Status.Stages[stage].Ended = endTime
}
if execution.Status.Started == "" {
execution.Status.Started = startTime
}
if execution.Status.Ended == "" {
execution.Status.Ended = endTime
}
if err := j.saveStepLogToMinio(execution, stage, step); err != nil {
return err
}
//clean waiting status of other stages/steps
for i := 0; i < len(execution.Status.Stages); i++ {
stage := &execution.Status.Stages[i]
if stage.State == utils.StateWaiting {
stage.State = ""
}
stepsize := len(execution.Status.Stages[i].Steps)
for j := 0; j < stepsize; j++ {
step := &stage.Steps[j]
if step.State == utils.StateWaiting {
step.State = ""
}
}
}
//abort concurrent building steps
for i := 0; i < len(execution.Status.Stages[stage].Steps); i++ {
if execution.Status.Stages[stage].Steps[i].State == utils.StateBuilding {
execution.Status.Stages[stage].Steps[i].State = utils.StateAborted
execution.Status.Stages[stage].Steps[i].Ended = endTime
if err := j.saveStepLogToMinio(execution, stage, i); err != nil {
return err
}
}
}
return nil
}
func buildingStep(execution *v3.PipelineExecution, stage int, step int, jenkinsStage Stage) {
startTime := time.Unix(jenkinsStage.StartTimeMillis/1000, 0).Format(time.RFC3339)
execution.Status.Stages[stage].Steps[step].State = utils.StateBuilding
if execution.Status.Stages[stage].Steps[step].Started == "" {
execution.Status.Stages[stage].Steps[step].Started = startTime
}
if execution.Status.Stages[stage].State == utils.StateWaiting {
execution.Status.Stages[stage].State = utils.StateBuilding
}
if execution.Status.Stages[stage].Started == "" {
execution.Status.Stages[stage].Started = startTime
}
if execution.Status.ExecutionState == utils.StateWaiting {
execution.Status.ExecutionState = utils.StateBuilding
}
if execution.Status.Started == "" {
execution.Status.Started = startTime
}
stageName := execution.Spec.PipelineConfig.Stages[stage].Name
message := fmt.Sprintf("Running '%s' stage", stageName)
v3.PipelineExecutionConditionBuilt.CreateUnknownIfNotExists(execution)
v3.PipelineExecutionConditionBuilt.Message(execution, message)
}
func skipStep(execution *v3.PipelineExecution, stage int, step int, jenkinsStage Stage) {
endTime := time.Unix((jenkinsStage.StartTimeMillis+jenkinsStage.DurationMillis)/1000, 0).Format(time.RFC3339)
execution.Status.Stages[stage].Steps[step].State = utils.StateSkipped
curStage := execution.Status.Stages[stage]
skipStage := true
for _, curStep := range curStage.Steps {
if curStep.State != utils.StateSkipped {
skipStage = false
}
}
if skipStage {
execution.Status.Stages[stage].State = utils.StateSkipped
} else if utils.IsStageSuccess(execution.Status.Stages[stage]) {
execution.Status.Stages[stage].State = utils.StateSuccess
execution.Status.Stages[stage].Ended = endTime
}
if stage == len(execution.Status.Stages)-1 &&
(execution.Status.Stages[stage].State == utils.StateSkipped || execution.Status.Stages[stage].State == utils.StateSuccess) {
execution.Status.ExecutionState = utils.StateSuccess
execution.Status.Ended = endTime
execution.Labels[utils.PipelineFinishLabel] = "true"
v3.PipelineExecutionConditionBuilt.True(execution)
}
}
func (j Engine) GetStepLog(execution *v3.PipelineExecution, stage int, step int) (string, error) {
if len(execution.Status.Stages) <= stage || len(execution.Status.Stages[stage].Steps) <= step {
return "", errors.New("invalid step index")
}
curStep := execution.Status.Stages[stage].Steps[step]
if curStep.State == utils.StateWaiting {
return "", nil
} else if curStep.State != utils.StateBuilding {
return j.getStepLogFromMinioStore(execution, stage, step)
}
return j.getStepLogFromJenkins(execution, stage, step)
}
func (j Engine) getStepLogFromJenkins(execution *v3.PipelineExecution, stage int, step int) (string, error) {
if len(execution.Status.Stages) <= stage || len(execution.Status.Stages[stage].Steps) <= step {
return "", errors.New("invalid step index")
}
jobName := getJobName(execution)
client, err := j.getJenkinsClient(execution)
if err != nil {
return "", err
}
info, err := client.getWFBuildInfo(jobName)
if err != nil {
return "", err
}
WFnodeID := ""
for _, jStage := range info.Stages {
if jStage.Name == fmt.Sprintf("step-%d-%d", stage, step) {
WFnodeID = jStage.ID
break
}
}
if WFnodeID == "" {
return "", errors.New("Error WF Node for the step not found")
}
WFnodeInfo, err := client.getWFNodeInfo(jobName, WFnodeID)
if err != nil {
return "", err
}
if len(WFnodeInfo.StageFlowNodes) < 1 {
return "", errors.New("Error step Node not found")
}
logNodeID := WFnodeInfo.StageFlowNodes[0].ID
logrus.Debugf("trying getWFNodeLog, %v, %v", jobName, logNodeID)
nodeLog, err := client.getWFNodeLog(jobName, logNodeID)
if err != nil {
return "", err
}
return nodeLog.Text, nil
}
func getJobName(execution *v3.PipelineExecution) string {
if execution == nil {
return ""
}
return fmt.Sprintf("%s%s", JenkinsJobPrefix, execution.Name)
}
func (j Engine) setCredential(client *Client, execution *v3.PipelineExecution, credentialID string) error {
if credentialID == "" {
return nil
}
ns, name := ref.Parse(credentialID)
souceCodeCredential, err := j.SourceCodeCredentialLister.Get(ns, name)
if err != nil {
return err
}
err = client.getCredential(credentialID)
if e, ok := err.(*httperror.APIError); !ok || e.Code.Status != http.StatusNotFound {
return err
}
//set credential when it is not exist
jenkinsCred := &Credential{}
jenkinsCred.Class = "com.cloudbees.plugins.credentials.impl.UsernamePasswordCredentialsImpl"
jenkinsCred.Scope = "GLOBAL"
jenkinsCred.ID = execution.Name
jenkinsCred.Username = souceCodeCredential.Spec.GitLoginName
jenkinsCred.Password = souceCodeCredential.Spec.AccessToken
bodyContent := map[string]interface{}{}
bodyContent["credentials"] = jenkinsCred
b, err := json.Marshal(bodyContent)
if err != nil {
return err
}
buff := bytes.NewBufferString("json=")
buff.Write(b)
return client.createCredential(buff.Bytes())
}
func translatePreparingMessage(log string) string {
log = strings.TrimRight(log, "\n")
lines := strings.Split(log, "\n")
message := lines[len(lines)-1]
if message == "" {
message = "Setting up executors"
} else if strings.Contains(message, " offline") {
message = "Waiting executors to be ready"
} else if strings.Contains(message, "Running") {
message = stripTags(message)
} else {
//unchanged
message = ""
}
return message
}
func stripTags(content string) string {
re, _ := regexp.Compile("\\<[\\S\\s]+?\\>")
stripped := re.ReplaceAllString(content, "")
return stripped
}
马建仓 AI 助手
尝试更多
代码解读
代码找茬
代码优化
Go
1
https://gitee.com/rancher/rancher.git
git@gitee.com:rancher/rancher.git
rancher
rancher
rancher
v2.1.9-rc3

搜索帮助

344bd9b3 5694891 D2dac590 5694891